Merge pull request 'The Go and the Node build modules, ssh-client on the SDK's key keeper, and gitea's read-only builder accounts (design 47 R4c-3)' (#389) from feat/r4c3-build-modules into main
This commit was merged in pull request #389.
This commit is contained in:
@@ -25,3 +25,4 @@ modules/mesh-files/cmd/mesh-files/mesh-files
|
||||
# The provisioner client built in place (issue 671).
|
||||
modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-client
|
||||
modules/brother/cmd/brother/brother
|
||||
modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go
|
||||
|
||||
@@ -160,8 +160,14 @@ func Check(root string) ([]Finding, error) {
|
||||
}
|
||||
dirs := map[string]string{}
|
||||
for _, r := range m.Resources {
|
||||
if r.ID != "" && r.Path != "" {
|
||||
switch {
|
||||
case r.ID != "" && r.Path != "":
|
||||
dirs[r.ID] = r.Path
|
||||
case r.ID != "" && r.Type == "directory":
|
||||
// A directory the mesh places for the module (`place`): where it lies is the node-engine's, but it is
|
||||
// the module's own, so a path under it belongs to this module and to no other's directory of the same
|
||||
// id (issue 1484: two runs-as modules each keep a .ssh in their own state directory).
|
||||
dirs[r.ID] = "/placed/" + m.Module + "/" + r.ID
|
||||
}
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
|
||||
@@ -130,3 +130,18 @@ func TestAnArchiveIsNotAContainerAndSameModuleIsFine(t *testing.T) {
|
||||
t.Fatalf("refused %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1484: a directory the mesh places for a module (no path, `place`) is that module's own: another module's
|
||||
// directory of the same id, and a path under it, are not inside it.
|
||||
func TestPlacedDirectoriesOfTwoModulesAreTheirOwn(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, name := range []string{"first", "second"} {
|
||||
manifest(t, root, name, mod(name,
|
||||
`{"id":"state","type":"directory","place":".","unknown":"program"}`,
|
||||
dir("ssh", "${dir:state}/.ssh", `,"unknown":"program"`),
|
||||
`{"id":"conf","type":"file","path":"${dir:ssh}/config"}`))
|
||||
}
|
||||
if found := check(t, root); len(found) != 0 {
|
||||
t.Fatalf("refused %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
+55
-6
@@ -5,7 +5,7 @@
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Every string of a manifest is read, wherever it stands, except prose and a file's text (Prose): a path, a
|
||||
// place, a volume. Exits 1 on a finding.
|
||||
// place, a volume; a path inside a directory of the module's own (`${dir:<id>}`) is not ~/.ssh. Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SSHModule is the one module allowed to name a path under ~/.ssh.
|
||||
@@ -60,7 +61,7 @@ func Check(root string) ([]Finding, error) {
|
||||
if module == SSHModule {
|
||||
continue
|
||||
}
|
||||
walk(m, "", func(key, value string) {
|
||||
walk(m, "", dirsOf(m), func(key, value string) {
|
||||
out = append(out, Finding{Module: module, Key: key, Value: value})
|
||||
})
|
||||
}
|
||||
@@ -68,7 +69,7 @@ func Check(root string) ([]Finding, error) {
|
||||
}
|
||||
|
||||
// walk calls found for every string under v, outside Prose, that names a path under ~/.ssh.
|
||||
func walk(v any, key string, found func(key, value string)) {
|
||||
func walk(v any, key string, dirs map[string]string, found func(key, value string)) {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(x))
|
||||
@@ -78,20 +79,68 @@ func walk(v any, key string, found func(key, value string)) {
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if !Prose[k] {
|
||||
walk(x[k], k, found)
|
||||
walk(x[k], k, dirs, found)
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, e := range x {
|
||||
walk(e, key, found)
|
||||
walk(e, key, dirs, found)
|
||||
}
|
||||
case string:
|
||||
if underSSH.MatchString(x) {
|
||||
// A path under a directory the mesh places for the module (${dir:<id>} of a directory with a place and no path,
|
||||
// as a runs-as module's home) is that account's own .ssh, not the operator's ~/.ssh that ssh-client sweeps
|
||||
// (issue 1484, ADR 0466 §4). Any other ${dir:<id>} is resolved to the path its resource declares and judged.
|
||||
if resolved, placed := resolveDir(x, dirs); !placed && underSSH.MatchString(resolved) {
|
||||
found(key, x)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var ofDir = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9_-]*)\}`)
|
||||
|
||||
// placedDir marks a directory the mesh places for the module: it has no path of its own.
|
||||
const placedDir = "\x00placed"
|
||||
|
||||
// dirsOf are the module's directories by id: the path each declares, or placedDir for one with a place and no path.
|
||||
func dirsOf(m any) map[string]string {
|
||||
dirs := map[string]string{}
|
||||
obj, _ := m.(map[string]any)
|
||||
list, _ := obj["resources"].([]any)
|
||||
for _, r := range list {
|
||||
res, _ := r.(map[string]any)
|
||||
id, _ := res["id"].(string)
|
||||
path, _ := res["path"].(string)
|
||||
switch {
|
||||
case id == "":
|
||||
case path != "":
|
||||
dirs[id] = path
|
||||
case res["type"] == "directory":
|
||||
dirs[id] = placedDir
|
||||
}
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
// resolveDir follows a leading ${dir:<id>} through the module's directories. It answers the path with the references
|
||||
// resolved, and whether it ends under a placed directory.
|
||||
func resolveDir(x string, dirs map[string]string) (string, bool) {
|
||||
for i := 0; i < 8; i++ {
|
||||
m := ofDir.FindStringSubmatch(x)
|
||||
if m == nil {
|
||||
return x, false
|
||||
}
|
||||
to, ok := dirs[m[1]]
|
||||
if !ok {
|
||||
return x, false
|
||||
}
|
||||
if to == placedDir {
|
||||
return x, true
|
||||
}
|
||||
x = to + strings.TrimPrefix(x, m[0])
|
||||
}
|
||||
return x, false
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
|
||||
@@ -37,6 +37,36 @@ func TestAnotherModuleNamingSSHIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1484: a runs-as module's own account has a .ssh of its own, inside its home (a directory of the module's);
|
||||
// that is not the operator's ~/.ssh, which only ssh-client names. A path outside any ${dir:} is still refused.
|
||||
func TestAModulesOwnDirectoryMayHoldADotSSH(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "builder", `{"module":"builder","resources":[{"id":"state","type":"directory","place":"."},{"id":"ssh-dir","type":"directory","path":"${dir:state}/.ssh"},{"path":"${dir:ssh-dir}/config"}]}`)
|
||||
manifest(t, root, "other", `{"module":"other","resources":[{"path":"${machine:account-home}/.ssh/config"}]}`)
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Module != "other" {
|
||||
t.Fatalf("refused %+v; want only other", found)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1484 review: only a directory the mesh places is the module's own home. A ${dir:} that resolves to the
|
||||
// operator's home is judged by the path it resolves to.
|
||||
func TestADirResolvingToTheOperatorsHomeIsStillRefused(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "escape", `{"module":"escape","resources":[{"id":"h","type":"directory","path":"${machine:account-home}","unknown":"module"},{"path":"${dir:h}/.ssh/config"}]}`)
|
||||
manifest(t, root, "placed", `{"module":"placed","resources":[{"id":"state","type":"directory","place":".","unknown":"program"},{"id":"s","type":"directory","path":"${dir:state}/.ssh"},{"path":"${dir:s}/config"}]}`)
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || found[0].Module != "escape" {
|
||||
t.Fatalf("refused %+v; want only escape", found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCatalogueHasNone(t *testing.T) {
|
||||
found, err := Check("../..")
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"gitea/internal/forgehttp"
|
||||
)
|
||||
|
||||
// Issue 1481 (novox/hq ADR 0466 §3): the build modules' keys on one forge user that reads and pushes nothing. The forge
|
||||
// is faked with the teams, members and keys it keeps, so a team's shape and a key's right can be read back.
|
||||
|
||||
type forgeKey struct {
|
||||
ID int64
|
||||
Title string
|
||||
Key string
|
||||
ReadOnly bool
|
||||
}
|
||||
|
||||
type teamForge struct {
|
||||
mu sync.Mutex
|
||||
users map[string]bool
|
||||
teams map[string]map[string]any // by name: the shape last sent
|
||||
ids map[string]int64
|
||||
members map[string][]string // by team name
|
||||
keys map[string][]forgeKey
|
||||
next int64
|
||||
}
|
||||
|
||||
func newTeamForge() *teamForge {
|
||||
return &teamForge{users: map[string]bool{}, teams: map[string]map[string]any{}, ids: map[string]int64{},
|
||||
members: map[string][]string{}, keys: map[string][]forgeKey{}, next: 10}
|
||||
}
|
||||
|
||||
var (
|
||||
teamRe = regexp.MustCompile(`^/api/v1/teams/(\d+)(?:/members/([^/]+))?$`)
|
||||
userKeyRe = regexp.MustCompile(`^/api/v1/(?:admin/)?users/([^/]+)/keys(?:/(\d+))?$`)
|
||||
)
|
||||
|
||||
func (f *teamForge) teamByID(id string) string {
|
||||
for name, n := range f.ids {
|
||||
if fmt.Sprint(n) == id {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (f *teamForge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
var in map[string]any
|
||||
_ = json.Unmarshal(body, &in)
|
||||
reply := func(code int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
if v != nil {
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
}
|
||||
p := r.URL.Path
|
||||
if m := userKeyRe.FindStringSubmatch(p); m != nil {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
out := []map[string]any{}
|
||||
for _, k := range f.keys[m[1]] {
|
||||
out = append(out, map[string]any{"id": k.ID, "title": k.Title, "key": k.Key})
|
||||
}
|
||||
reply(200, out)
|
||||
case http.MethodPost:
|
||||
f.next++
|
||||
ro, _ := in["read_only"].(bool)
|
||||
f.keys[m[1]] = append(f.keys[m[1]], forgeKey{ID: f.next, Title: in["title"].(string), Key: in["key"].(string), ReadOnly: ro})
|
||||
reply(201, map[string]any{"id": f.next})
|
||||
case http.MethodDelete:
|
||||
var kept []forgeKey
|
||||
for _, k := range f.keys[m[1]] {
|
||||
if fmt.Sprint(k.ID) != m[2] {
|
||||
kept = append(kept, k)
|
||||
}
|
||||
}
|
||||
f.keys[m[1]] = kept
|
||||
reply(204, nil)
|
||||
}
|
||||
return
|
||||
}
|
||||
if m := teamRe.FindStringSubmatch(p); m != nil {
|
||||
name := f.teamByID(m[1])
|
||||
switch {
|
||||
case name == "":
|
||||
reply(404, map[string]any{"message": "no team"})
|
||||
case r.Method == http.MethodPatch && m[2] == "":
|
||||
f.teams[name] = in
|
||||
reply(200, map[string]any{})
|
||||
case r.Method == http.MethodPut:
|
||||
for _, u := range f.members[name] {
|
||||
if u == m[2] {
|
||||
reply(204, nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
f.members[name] = append(f.members[name], m[2])
|
||||
reply(204, nil)
|
||||
default:
|
||||
http.Error(w, "unexpected "+r.Method+" "+p, 400)
|
||||
}
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(p, "/api/v1/users/"):
|
||||
if !f.users[strings.TrimPrefix(p, "/api/v1/users/")] {
|
||||
reply(404, map[string]any{"message": "no"})
|
||||
return
|
||||
}
|
||||
reply(200, map[string]any{"active": true})
|
||||
case r.Method == http.MethodPost && p == "/api/v1/admin/users":
|
||||
f.users[in["username"].(string)] = true
|
||||
reply(201, map[string]any{})
|
||||
case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodGet:
|
||||
out := []map[string]any{}
|
||||
for name, id := range f.ids {
|
||||
out = append(out, map[string]any{"id": id, "name": name})
|
||||
}
|
||||
reply(200, out)
|
||||
case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodPost:
|
||||
f.next++
|
||||
name := in["name"].(string)
|
||||
f.ids[name], f.teams[name] = f.next, in
|
||||
reply(201, map[string]any{"id": f.next})
|
||||
default:
|
||||
http.Error(w, "unexpected "+r.Method+" "+p, 400)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *teamForge) titles(user string) []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := []string{}
|
||||
for _, k := range f.keys[user] {
|
||||
out = append(out, k.Title)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type teamRig struct {
|
||||
t *testing.T
|
||||
bus *fakeBus
|
||||
forge *teamForge
|
||||
svc *Service
|
||||
runner *fakeRunner
|
||||
}
|
||||
|
||||
func newTeamRig(t *testing.T) *teamRig {
|
||||
t.Helper()
|
||||
forge := newTeamForge()
|
||||
srv := httptest.NewServer(forge)
|
||||
t.Cleanup(srv.Close)
|
||||
dir := t.TempDir()
|
||||
pw, host, origins := filepath.Join(dir, "pw"), filepath.Join(dir, "host"), filepath.Join(dir, "origins")
|
||||
for p, c := range map[string]string{pw: "x\n", host: "novox.internal\n", origins: "https://git.novox.be\n"} {
|
||||
if err := os.WriteFile(p, []byte(c), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
runner := &fakeRunner{out: scanned}
|
||||
svc := NewService(Config{Node: "novox", HostFile: host, OriginsFile: origins, Port: 222,
|
||||
Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}, Runner: runner, Log: func(string, ...any) {}})
|
||||
svc.loaded = true
|
||||
svc.loadedFrom["mesh-build-go"], svc.loadedFrom["mesh-build-node"] = true, true
|
||||
return &teamRig{t: t, bus: newBus(), forge: forge, svc: svc, runner: runner}
|
||||
}
|
||||
|
||||
func (r *teamRig) pass() []provider.Outcome {
|
||||
r.t.Helper()
|
||||
out, err := r.svc.Provider(r.bus).Pass(context.Background())
|
||||
if err != nil {
|
||||
r.t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// bind makes module on node bind gitea's forge-account in the assignments mirror.
|
||||
func (r *teamRig) bind(node, module string) {
|
||||
r.bus.set(provider.AssignmentsBucket, node+"."+module, []byte(fmt.Sprintf(
|
||||
`{"node":%q,"module":%q,"provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s"}]}}`,
|
||||
node, module, strings.ReplaceAll(node+"_"+module, "-", "_"))))
|
||||
}
|
||||
|
||||
func (r *teamRig) unbind(node, module string) {
|
||||
r.bus.mu.Lock()
|
||||
delete(r.bus.buckets[provider.AssignmentsBucket], node+"."+module)
|
||||
r.bus.mu.Unlock()
|
||||
}
|
||||
|
||||
// key reports a key in module's state keys, of the kind and for the account given.
|
||||
func (r *teamRig) key(module, node, account, kind, public string) {
|
||||
b, _ := json.Marshal(map[string]string{"node": node, "account": account, "kind": kind, "public": public,
|
||||
"fingerprint": "SHA256:x", "at": "2026-10-20T10:00:00Z"})
|
||||
r.svc.SourceKeyChanged(module, node+"."+account, b)
|
||||
}
|
||||
|
||||
const (
|
||||
pubGo = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG"
|
||||
pubNode = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN"
|
||||
pubOps = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO"
|
||||
)
|
||||
|
||||
// 1481/AC1: an entry from mesh-build-go on ace: the forge user nox-builder and the team builders are ensured, the team's
|
||||
// units are code, read only, on every repository, and the entry's values name nox-builder. The key is registered
|
||||
// read-only under its title.
|
||||
func TestABuildModulesEntryIsServedAsNoxBuilderInAReadOnlyTeam(t *testing.T) {
|
||||
r := newTeamRig(t)
|
||||
r.bind("ace", "mesh-build-go")
|
||||
r.key("mesh-build-go", "ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)")
|
||||
r.pass()
|
||||
|
||||
if !r.forge.users["nox-builder"] || r.forge.users["nox-agent"] {
|
||||
t.Errorf("users = %v, want nox-builder alone", r.forge.users)
|
||||
}
|
||||
shape := r.forge.teams["builders"]
|
||||
if shape == nil {
|
||||
t.Fatalf("no team builders: %v", r.forge.teams)
|
||||
}
|
||||
units, _ := json.Marshal(shape["units_map"])
|
||||
if shape["permission"] != "read" || shape["includes_all_repositories"] != true || shape["can_create_org_repo"] != false || string(units) != `{"repo.code":"read"}` {
|
||||
t.Errorf("the team's shape = %v (units %s), want code read on every repository and no other right", shape, units)
|
||||
}
|
||||
if got := strings.Join(r.forge.members["builders"], ","); got != "nox-builder" || len(r.forge.members["agents"]) != 0 {
|
||||
t.Errorf("members = %v", r.forge.members)
|
||||
}
|
||||
keys := r.forge.keys["nox-builder"]
|
||||
if len(keys) != 1 || keys[0].Title != "mesh:ace.mesh-build-go" || !keys[0].ReadOnly {
|
||||
t.Errorf("nox-builder's keys = %+v, want one read-only key titled mesh:ace.mesh-build-go", keys)
|
||||
}
|
||||
e := r.bus.entry(t, "ace.mesh-build-go.forge-account")
|
||||
if e.Values["user"] != "nox-builder" || e.Values["ssh_alias"] != "mesh-forge" || e.Values["host_key"] != "ssh-ed25519 AAAAC3HOSTKEY" {
|
||||
t.Errorf("values = %v", e.Values)
|
||||
}
|
||||
}
|
||||
|
||||
// 1481/AC2: an entry from ssh-client is what it was: nox-agent, the agents team, a key that may write. (The existing
|
||||
// tests of the provider pass unchanged.)
|
||||
func TestAnSshClientEntryIsStillTheAgentsWithAKeyThatMayWrite(t *testing.T) {
|
||||
r := newTeamRig(t)
|
||||
// the agents team exists, as the forge has it
|
||||
r.forge.ids["agents"], r.forge.teams["agents"] = 5, map[string]any{}
|
||||
r.bind("g14", "ssh-client")
|
||||
r.key("ssh-client", "g14", "nox", "machine", pubA+" nox@g14 (mesh)")
|
||||
r.pass()
|
||||
if e := r.bus.entry(t, "g14.ssh-client.forge-account"); e.Values["user"] != "nox-agent" {
|
||||
t.Errorf("values = %v", e.Values)
|
||||
}
|
||||
keys := r.forge.keys["nox-agent"]
|
||||
if len(keys) != 1 || keys[0].Title != "mesh:g14.nox" || keys[0].ReadOnly {
|
||||
t.Errorf("nox-agent's keys = %+v, want one key that may write", keys)
|
||||
}
|
||||
if r.forge.teams["builders"] != nil || r.forge.users["nox-builder"] {
|
||||
t.Errorf("a machine's entry made the builders' account or team")
|
||||
}
|
||||
if got := strings.Join(r.forge.members["agents"], ","); got != "nox-agent" {
|
||||
t.Errorf("agents = %v", r.forge.members)
|
||||
}
|
||||
}
|
||||
|
||||
// 1481/AC3: a builder entry gone: only the keys titled mesh:<node>.mesh-build-go are removed; the user and the team
|
||||
// stay, and so do the other build module's key on that machine and a key of the operator's.
|
||||
func TestABuilderEntryGoneRemovesItsOwnKeyAndNothingElse(t *testing.T) {
|
||||
r := newTeamRig(t)
|
||||
r.bind("ace", "mesh-build-go")
|
||||
r.bind("ace", "mesh-build-node")
|
||||
r.key("mesh-build-go", "ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)")
|
||||
r.key("mesh-build-node", "ace", "mesh-build-node", "builder", pubNode+" mesh-build-node@ace (mesh)")
|
||||
r.pass()
|
||||
r.forge.mu.Lock()
|
||||
r.forge.keys["nox-builder"] = append(r.forge.keys["nox-builder"], forgeKey{ID: 99, Title: "my own", Key: pubOps})
|
||||
r.forge.mu.Unlock()
|
||||
if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-go,mesh:ace.mesh-build-node,my own" {
|
||||
t.Fatalf("before: %s", got)
|
||||
}
|
||||
r.unbind("ace", "mesh-build-go")
|
||||
r.pass()
|
||||
if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-node,my own" {
|
||||
t.Errorf("after: %s, want only the go module's key gone", got)
|
||||
}
|
||||
if !r.forge.users["nox-builder"] || r.forge.teams["builders"] == nil || len(r.forge.members["builders"]) != 1 {
|
||||
t.Errorf("the user or the team went: users %v, members %v", r.forge.users, r.forge.members)
|
||||
}
|
||||
if r.bus.has("ace.mesh-build-go.forge-account") {
|
||||
t.Errorf("the entry stayed")
|
||||
}
|
||||
}
|
||||
|
||||
// A build module and ssh-client on one machine each have their own entry, keys, user and team.
|
||||
func TestABuildModuleAndSshClientOnOneMachineDoNotShareAnything(t *testing.T) {
|
||||
r := newTeamRig(t)
|
||||
r.forge.ids["agents"], r.forge.teams["agents"] = 5, map[string]any{}
|
||||
r.bind("ace", "ssh-client")
|
||||
r.bind("ace", "mesh-build-go")
|
||||
r.key("ssh-client", "ace", "nox", "machine", pubA+" nox@ace (mesh)")
|
||||
r.key("mesh-build-go", "ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)")
|
||||
r.pass()
|
||||
if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "mesh:ace.nox" {
|
||||
t.Errorf("nox-agent: %s", got)
|
||||
}
|
||||
if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-go" {
|
||||
t.Errorf("nox-builder: %s", got)
|
||||
}
|
||||
// ssh-client's entry gone: nox-builder's key stays.
|
||||
r.unbind("ace", "ssh-client")
|
||||
r.pass()
|
||||
if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-go" {
|
||||
t.Errorf("nox-builder after ssh-client left: %s", got)
|
||||
}
|
||||
if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "" {
|
||||
t.Errorf("nox-agent after ssh-client left: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A build module's entry waits for its own module's builder key: a machine key of the same node, a key of the wrong
|
||||
// kind in the build module's state, or the other build module's key is none.
|
||||
func TestABuildModulesEntryWaitsForItsOwnBuilderKey(t *testing.T) {
|
||||
r := newTeamRig(t)
|
||||
r.bind("ace", "mesh-build-go")
|
||||
r.key("ssh-client", "ace", "nox", "machine", pubA) // a machine's key
|
||||
r.key("mesh-build-go", "ace", "mesh-build-go", "machine", pubGo) // the wrong kind in the build module's state
|
||||
r.key("mesh-build-node", "ace", "mesh-build-node", "builder", pubNode) // the other build module's
|
||||
out := r.pass()
|
||||
if r.bus.has("ace.mesh-build-go.forge-account") || len(r.forge.users) != 0 || len(r.forge.teams) != 0 {
|
||||
t.Fatalf("something was made without a builder key: users %v teams %v", r.forge.users, r.forge.teams)
|
||||
}
|
||||
if len(out) != 1 || !strings.Contains(out[0].Why, "mesh-build-go") || !strings.Contains(out[0].Why, "builder key") {
|
||||
t.Errorf("the wait should name the module and the key: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheKindOfAModulesKeyAndItsForgeUser(t *testing.T) {
|
||||
for module, want := range map[string]string{"mesh-build-go": "nox-builder", "mesh-build-node": "nox-builder", "ssh-client": "nox-agent", "anything-else": "nox-agent"} {
|
||||
got, err := (&Service{}).valuesForUser("ace", module)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("%s: %q, %v; want %q", module, got, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,10 @@
|
||||
// For each machine whose ssh-client binds the provision and has reported its public key (ssh-client's state `keys`),
|
||||
// it makes the machine's agent the one forge user nox-agent (in the agents team; ADR 0410), registers the key for
|
||||
// it, and serves where and as whom to reach the forge over ssh: the user, the forge machine's mesh name and port, its
|
||||
// host key and the two url roots. A public key is no secret, so the provision has no credential.
|
||||
// host key and the two url roots. A public key is no secret, so the provision has no credential. For each build
|
||||
// module that binds it and has reported its builder key (its own state `keys`), it does the same as the forge user
|
||||
// nox-builder, in the builders team that reads the code of every repository and nothing else, with the key read-only
|
||||
// (ADR 0466 §3).
|
||||
//
|
||||
// stdout is the MCP channel; it says what it says on stderr, and never the admin's password.
|
||||
package main
|
||||
@@ -22,6 +25,7 @@ import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"gitea/internal/forgeaccount"
|
||||
"gitea/internal/forgehttp"
|
||||
)
|
||||
|
||||
@@ -42,11 +46,12 @@ func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte,
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// stateKeys is ssh-client's state `keys` over the runtime (the manifest's `reads` grants it).
|
||||
type stateKeys struct{}
|
||||
// stateKeys is the state `keys` of one module over the runtime (the manifest's `reads` grants it): ssh-client's for
|
||||
// the machines' keys, a build module's for the builders' (ADR 0466 §3).
|
||||
type stateKeys struct{ module string }
|
||||
|
||||
func (stateKeys) Watch(handle func(key, op string, value []byte, current bool)) error {
|
||||
return stdio.State("ssh-client.keys").Watch("", func(c stdio.StateChange) error {
|
||||
func (k stateKeys) Watch(handle func(key, op string, value []byte, current bool)) error {
|
||||
return stdio.State(k.module+".keys").Watch("", func(c stdio.StateChange) error {
|
||||
handle(c.Key, c.Op, c.Value, c.Current)
|
||||
return nil
|
||||
})
|
||||
@@ -106,14 +111,18 @@ func main() {
|
||||
}
|
||||
}
|
||||
svc.OnKeys = func(string) { go pass() }
|
||||
go func() {
|
||||
if err := svc.WatchKeys(ctx, stateKeys{}); err != nil {
|
||||
say("the machines' keys are not watched: %v", err)
|
||||
return
|
||||
}
|
||||
say("watching the machines' keys")
|
||||
pass()
|
||||
}()
|
||||
// Each module's keys are watched on their own, and a module whose state never answers (a build module that is not
|
||||
// installed) holds back its own entries only.
|
||||
for _, module := range append([]string{SSHClient}, forgeaccount.BuildModules...) {
|
||||
go func() {
|
||||
if err := svc.WatchSource(ctx, module, stateKeys{module}); err != nil {
|
||||
say("the keys of %s are not watched: %v", module, err)
|
||||
return
|
||||
}
|
||||
say("watching the keys of %s", module)
|
||||
pass()
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
svc.KeepRunning(ctx, p.Run)
|
||||
}()
|
||||
|
||||
@@ -64,8 +64,11 @@ type Service struct {
|
||||
|
||||
mu sync.Mutex
|
||||
keys map[string]machineKey
|
||||
loaded bool // the machines' keys have been read: until then a node with none has not reported none
|
||||
known string // the forge's host key line, as the machines write it into known_hosts
|
||||
loaded bool // the machines' keys have been read: until then a node with none has not reported none
|
||||
// loadedFrom says of each build module's state that its keys have been read, as loaded says of ssh-client's: a build
|
||||
// module's state that never answers (the module is not installed) holds back its own entries and no other.
|
||||
loadedFrom map[string]bool
|
||||
known string // the forge's host key line, as the machines write it into known_hosts
|
||||
|
||||
bound provider.Bus // the bus the entries are written to (set by Provider)
|
||||
|
||||
@@ -76,14 +79,40 @@ type Service struct {
|
||||
|
||||
type machineKey struct {
|
||||
Node, Account, Kind, Public string
|
||||
// Source is the module whose state the key was read from: ssh-client's for a machine's, a build module's for a
|
||||
// builder's. Empty is ssh-client's.
|
||||
Source string
|
||||
}
|
||||
|
||||
// SSHClient is the module whose `keys` state holds the machines' keys (kind machine); the build modules' states hold
|
||||
// the builders' (kind builder, ADR 0466 §2).
|
||||
const SSHClient = "ssh-client"
|
||||
|
||||
// sourceOf is the module a stored key was read from.
|
||||
func (k machineKey) sourceOf() string {
|
||||
if k.Source == "" {
|
||||
return SSHClient
|
||||
}
|
||||
return k.Source
|
||||
}
|
||||
|
||||
// storageKey is the key a source's key is kept under: a machine's as it is, a builder's under its module's name.
|
||||
func storageKey(source, key string) string {
|
||||
if source == "" || source == SSHClient {
|
||||
return key
|
||||
}
|
||||
return source + "/" + key
|
||||
}
|
||||
|
||||
// kindOf is the kind of key a consumer module's entry is made for.
|
||||
func kindOf(module string) string { return forgeaccount.KindOfModule(module) }
|
||||
|
||||
// NewService is the service over its environment.
|
||||
func NewService(cfg Config) *Service {
|
||||
if cfg.Log == nil {
|
||||
cfg.Log = func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[gitea forge-account] "+f+"\n", a...) }
|
||||
}
|
||||
return &Service{cfg: cfg, keys: map[string]machineKey{}, letGo: map[string]bool{}, Pause: time.Sleep,
|
||||
return &Service{cfg: cfg, keys: map[string]machineKey{}, letGo: map[string]bool{}, loadedFrom: map[string]bool{}, Pause: time.Sleep,
|
||||
accounts: &forgeaccount.Accounts{Forge: cfg.Forge, Log: cfg.Log}}
|
||||
}
|
||||
|
||||
@@ -103,38 +132,67 @@ func (s *Service) Provider(bus provider.Bus) *provider.Provider {
|
||||
}
|
||||
|
||||
// errNoKey is a consumer whose machine has reported no key yet: nothing is made, and the client says it waits.
|
||||
type errNoKey struct{ node string }
|
||||
type errNoKey struct{ node, module string }
|
||||
|
||||
func (e errNoKey) Error() string {
|
||||
if kind := kindOf(e.module); kind == "builder" {
|
||||
return "no key from " + e.node + " yet: its " + e.module + " has not reported a builder key, so there is no " +
|
||||
"entry to make"
|
||||
}
|
||||
return "no key from " + e.node + " yet: its ssh-client has not reported a machine key, so there is no account " +
|
||||
"and no entry to make"
|
||||
}
|
||||
|
||||
// sourceOfModule is the module whose state holds the keys a consumer module's entry is made from: a build module's
|
||||
// own, ssh-client's for any other consumer.
|
||||
func sourceOfModule(module string) string {
|
||||
if kindOf(module) == "builder" {
|
||||
return module
|
||||
}
|
||||
return SSHClient
|
||||
}
|
||||
|
||||
// scopeFor is the title prefix of the keys one entry registered: every key of a machine for a machine's entry, one
|
||||
// build module's key for a builder's (nox-builder holds the keys of every build module of the mesh).
|
||||
func scopeFor(kind, node, module string) string {
|
||||
if kind == "builder" {
|
||||
return forgeaccount.BuilderScope(node, module)
|
||||
}
|
||||
return forgeaccount.MachineScope(node)
|
||||
}
|
||||
|
||||
// Make brings one consumer's account, its team membership and its keys into being, as the machine reported them. It
|
||||
// leaves what is as asked alone, so it is asked again freely.
|
||||
func (s *Service) Make(ctx context.Context, c provider.Consumer) error {
|
||||
s.making.Lock()
|
||||
defer s.making.Unlock()
|
||||
node := c.Entry.Consumer
|
||||
user := forgeaccount.ForgeUser(node, "machine")
|
||||
module, kind := c.Entry.Module, kindOf(c.Entry.Module)
|
||||
user := forgeaccount.ForgeUser(node, kind)
|
||||
if user == "" {
|
||||
return fmt.Errorf("%q is not a machine's name an account can be named for", node)
|
||||
}
|
||||
source := sourceOfModule(module)
|
||||
s.mu.Lock()
|
||||
loaded := s.loaded
|
||||
if source != SSHClient {
|
||||
loaded = s.loadedFrom[source]
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !loaded {
|
||||
// After a restart the first pass can come before the keys arrive: nothing is removed or deleted on that.
|
||||
return errors.New("the machines' keys are not read yet, so nothing is made, removed or deleted")
|
||||
return fmt.Errorf("the keys of %s are not read yet, so nothing is made, removed or deleted", source)
|
||||
}
|
||||
have := s.keysOf(node)
|
||||
have := s.keysFrom(source, node, kind)
|
||||
if len(have) == 0 {
|
||||
if err := s.releasePerNodeUser(ctx, node); err != nil {
|
||||
return err
|
||||
if kind == "machine" {
|
||||
if err := s.releasePerNodeUser(ctx, node); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// A key the machine no longer reports is no longer wanted (ADR 0400 §2): the mesh's keys go, the operator's
|
||||
// stay, and the entry that served the machine goes with them.
|
||||
if _, err := s.accounts.RemoveKeysTitled(ctx, user, forgeaccount.MachineScope(node)); err != nil {
|
||||
if _, err := s.accounts.RemoveKeysTitled(ctx, user, scopeFor(kind, node, module)); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.bound != nil {
|
||||
@@ -142,7 +200,7 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return errNoKey{node}
|
||||
return errNoKey{node: node, module: module}
|
||||
}
|
||||
if _, err := s.hostName(); err != nil {
|
||||
return err
|
||||
@@ -154,20 +212,23 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error {
|
||||
if err := s.accounts.EnsureAccount(ctx, user); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.accounts.EnsureInTeam(ctx, user); err != nil {
|
||||
if err := s.accounts.EnsureInTeamOf(ctx, forgeaccount.TeamOfUser(user), user); err != nil {
|
||||
return err
|
||||
}
|
||||
var wanted []forgeaccount.WantedKey
|
||||
for _, k := range have {
|
||||
wanted = append(wanted, forgeaccount.WantedKey{Title: forgeaccount.KeyTitle(k.Node, k.Account), Key: k.Public})
|
||||
// A builder's key may read and never push (ADR 0466 §3): the key says it beside the team that does.
|
||||
wanted = append(wanted, forgeaccount.WantedKey{Title: forgeaccount.KeyTitle(k.Node, k.Account), Key: k.Public, ReadOnly: kind == "builder"})
|
||||
}
|
||||
// The forge holds one public key on one user only, so the key leaves agent-<node> before it joins nox-agent. That
|
||||
// is done here, after everything that can fail before it, so the window in which the machine's key is on no user
|
||||
// is only the span between these two calls; if the add fails, the next pass (at most ten minutes) puts it back.
|
||||
if err := s.releasePerNodeUser(ctx, node); err != nil {
|
||||
return err
|
||||
if kind == "machine" {
|
||||
if err := s.releasePerNodeUser(ctx, node); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := s.accounts.EnsureKeys(ctx, user, forgeaccount.MachineScope(node), wanted); err != nil {
|
||||
if err := s.accounts.EnsureKeys(ctx, user, scopeFor(kind, node, module), wanted); err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
@@ -176,20 +237,25 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Gone removes the keys the mesh registered for a machine that no longer binds the forge. The account stays: it may
|
||||
// Gone removes the keys the mesh registered for a machine (or, for a build module, the module's key on that machine)
|
||||
// that no longer binds the forge. The account stays: it may
|
||||
// hold what the machine's agent wrote, and the keys of the operator's own that were added by hand are not touched
|
||||
// (ADR 0399 §2).
|
||||
func (s *Service) Gone(ctx context.Context, e provider.Entry) error {
|
||||
s.making.Lock()
|
||||
defer s.making.Unlock()
|
||||
user := forgeaccount.ForgeUser(e.Consumer, "machine")
|
||||
kind := kindOf(e.Module)
|
||||
user := forgeaccount.ForgeUser(e.Consumer, kind)
|
||||
if user == "" {
|
||||
return nil
|
||||
}
|
||||
if err := s.releasePerNodeUser(ctx, e.Consumer); err != nil {
|
||||
return err
|
||||
if kind == "machine" {
|
||||
if err := s.releasePerNodeUser(ctx, e.Consumer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err := s.accounts.RemoveKeysTitled(ctx, user, forgeaccount.MachineScope(e.Consumer))
|
||||
// A builder's entry gone removes that entry's key only: the user and the team serve the other machines.
|
||||
_, err := s.accounts.RemoveKeysTitled(ctx, user, scopeFor(kind, e.Consumer, e.Module))
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -220,17 +286,24 @@ func (s *Service) releasePerNodeUser(ctx context.Context, node string) error {
|
||||
}
|
||||
|
||||
// KeyChanged takes in one machine key as the bus keeps it, and tells OnKeys its node.
|
||||
func (s *Service) KeyChanged(key string, value []byte) {
|
||||
if node := s.store(key, value); node != "" && s.OnKeys != nil {
|
||||
func (s *Service) KeyChanged(key string, value []byte) { s.SourceKeyChanged(SSHClient, key, value) }
|
||||
|
||||
// KeyGone forgets one machine key, and tells OnKeys its node.
|
||||
func (s *Service) KeyGone(key string) { s.SourceKeyGone(SSHClient, key) }
|
||||
|
||||
// SourceKeyChanged takes in one key of the state `keys` of the module source (ssh-client's, or a build module's), and
|
||||
// tells OnKeys its node.
|
||||
func (s *Service) SourceKeyChanged(source, key string, value []byte) {
|
||||
if node := s.storeFrom(source, key, value); node != "" && s.OnKeys != nil {
|
||||
s.OnKeys(node)
|
||||
}
|
||||
}
|
||||
|
||||
// KeyGone forgets one machine key, and tells OnKeys its node.
|
||||
func (s *Service) KeyGone(key string) {
|
||||
// SourceKeyGone forgets one key of the module source's state, and tells OnKeys its node.
|
||||
func (s *Service) SourceKeyGone(source, key string) {
|
||||
node, _, _ := strings.Cut(key, ".")
|
||||
s.mu.Lock()
|
||||
delete(s.keys, key)
|
||||
delete(s.keys, storageKey(source, key))
|
||||
s.mu.Unlock()
|
||||
if s.OnKeys != nil {
|
||||
s.OnKeys(node)
|
||||
@@ -240,24 +313,38 @@ func (s *Service) KeyGone(key string) {
|
||||
// store keeps a machine key that reads as one: its node and account the key's own, and a public half that is a
|
||||
// public key line and nothing private. It answers the node, or "" for a value it did not keep (and drops what was
|
||||
// kept under that key, since the key now says something else).
|
||||
func (s *Service) store(key string, value []byte) string {
|
||||
func (s *Service) store(key string, value []byte) string { return s.storeFrom(SSHClient, key, value) }
|
||||
|
||||
// storeFrom is store for a key of the module source's state.
|
||||
func (s *Service) storeFrom(source, key string, value []byte) string {
|
||||
node, account, _ := strings.Cut(key, ".")
|
||||
stored := storageKey(source, key)
|
||||
var k machineKey
|
||||
var raw struct {
|
||||
Node, Account, Kind, Public string
|
||||
}
|
||||
err := json.Unmarshal(value, &raw)
|
||||
k = machineKey(raw)
|
||||
k = machineKey{Node: raw.Node, Account: raw.Account, Kind: raw.Kind, Public: raw.Public}
|
||||
if source != SSHClient {
|
||||
k.Source = source
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err != nil || k.Node != node || k.Account != account || !publicKeyLine(k.Public) {
|
||||
if _, had := s.keys[key]; had {
|
||||
delete(s.keys, key)
|
||||
}
|
||||
s.cfg.Log("the key %s is not a machine's public key; it is not registered", key)
|
||||
if want := kindOf(source); err == nil && k.Kind != want && k.Node == node && k.Account == account {
|
||||
// Each state holds one kind: ssh-client's the machines', a build module's the builders' (ADR 0466 §2). The other
|
||||
// kind in either is not this entry's to serve, and is said.
|
||||
delete(s.keys, stored)
|
||||
s.cfg.Log("the key %s in the state keys of %s is of kind %q, not %q; it is ignored", key, source, k.Kind, want)
|
||||
return ""
|
||||
}
|
||||
s.keys[key] = k
|
||||
if err != nil || k.Node != node || k.Account != account || !publicKeyLine(k.Public) {
|
||||
if _, had := s.keys[stored]; had {
|
||||
delete(s.keys, stored)
|
||||
}
|
||||
s.cfg.Log("the key %s of %s is not a public key that can be registered; it is not registered", key, source)
|
||||
return ""
|
||||
}
|
||||
s.keys[stored] = k
|
||||
return node
|
||||
}
|
||||
|
||||
@@ -267,12 +354,15 @@ func publicKeyLine(line string) bool {
|
||||
}
|
||||
|
||||
// keysOf are the machine keys one node reported, by key.
|
||||
func (s *Service) keysOf(node string) []machineKey {
|
||||
func (s *Service) keysOf(node string) []machineKey { return s.keysFrom(SSHClient, node, "machine") }
|
||||
|
||||
// keysFrom are the keys of the kind that one node reported in the module source's state, by key.
|
||||
func (s *Service) keysFrom(source, node, kind string) []machineKey {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var names []string
|
||||
for key, k := range s.keys {
|
||||
if k.Node == node && k.Kind == "machine" && strings.HasPrefix(key, node+".") {
|
||||
if k.Node == node && k.Kind == kind && k.sourceOf() == source && strings.HasPrefix(key, storageKey(source, node+".")) {
|
||||
names = append(names, key)
|
||||
}
|
||||
}
|
||||
@@ -284,27 +374,37 @@ func (s *Service) keysOf(node string) []machineKey {
|
||||
return out
|
||||
}
|
||||
|
||||
// WatchKeys keeps the machines' keys as the bus holds them, asking again until the runtime answers (the channel to it
|
||||
// opens as the bundle starts, and the state of ssh-client may arrive after this one). A change after the first
|
||||
// values is told to OnKeys; the first values are not, since one pass answers them all.
|
||||
// WatchKeys keeps the machines' keys as the bus holds them (ssh-client's state `keys`), asking again until the runtime
|
||||
// answers (the channel to it opens as the bundle starts, and the state of ssh-client may arrive after this one). A
|
||||
// change after the first values is told to OnKeys; the first values are not, since one pass answers them all.
|
||||
func (s *Service) WatchKeys(ctx context.Context, src KeySource) error {
|
||||
return s.WatchSource(ctx, SSHClient, src)
|
||||
}
|
||||
|
||||
// WatchSource is WatchKeys for the state `keys` of any module whose keys the provider registers: ssh-client's, or a
|
||||
// build module's (ADR 0466 §3). Once its first values are in, the entries made from them are no longer held back.
|
||||
func (s *Service) WatchSource(ctx context.Context, source string, src KeySource) error {
|
||||
waits := []time.Duration{2 * time.Second, 5 * time.Second, 10 * time.Second, 30 * time.Second}
|
||||
for attempt := 0; ; attempt++ {
|
||||
err := src.Watch(func(key, op string, value []byte, current bool) {
|
||||
switch {
|
||||
case op != "put" && current:
|
||||
case op != "put":
|
||||
s.KeyGone(key)
|
||||
s.SourceKeyGone(source, key)
|
||||
case current:
|
||||
s.store(key, value)
|
||||
s.storeFrom(source, key, value)
|
||||
default:
|
||||
s.KeyChanged(key, value)
|
||||
s.SourceKeyChanged(source, key, value)
|
||||
}
|
||||
})
|
||||
if err == nil {
|
||||
// The watch answers once the current values are all handled (stdio KeptState.Watch).
|
||||
s.mu.Lock()
|
||||
s.loaded = true
|
||||
if source == SSHClient {
|
||||
s.loaded = true
|
||||
} else {
|
||||
s.loadedFrom[source] = true
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
@@ -312,7 +412,7 @@ func (s *Service) WatchKeys(ctx context.Context, src KeySource) error {
|
||||
if attempt < len(waits) {
|
||||
pause = waits[attempt]
|
||||
}
|
||||
s.cfg.Log("the machines' keys cannot be watched yet (%v); asking again in %s", err, pause)
|
||||
s.cfg.Log("the keys of %s cannot be watched yet (%v); asking again in %s", source, err, pause)
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
@@ -377,9 +477,18 @@ func knownHostLine(scanned, host string, port int) (string, error) {
|
||||
return "", errors.New("the forge's sshd gave no ed25519 host key")
|
||||
}
|
||||
|
||||
// valuesForUser is the forge user a consumer module's entry is served as.
|
||||
func (s *Service) valuesForUser(node, module string) (string, error) {
|
||||
user := forgeaccount.ForgeUser(node, kindOf(module))
|
||||
if user == "" {
|
||||
return "", fmt.Errorf("%q is not a machine's name an account can be named for", node)
|
||||
}
|
||||
return user, nil
|
||||
}
|
||||
|
||||
// valuesFor are the values served to one machine's consumer (ADR 0400 §3, §4).
|
||||
func (s *Service) valuesFor(node string) (map[string]any, error) {
|
||||
user := forgeaccount.ForgeUser(node, "machine")
|
||||
func (s *Service) valuesFor(node, module string) (map[string]any, error) {
|
||||
user := forgeaccount.ForgeUser(node, kindOf(module))
|
||||
host, err := s.hostName()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -421,7 +530,7 @@ func (b *valuesBus) Put(ctx context.Context, bucket, key string, value []byte) e
|
||||
return err
|
||||
}
|
||||
if e.Refused == "" {
|
||||
extra, err := b.s.valuesFor(e.Consumer)
|
||||
extra, err := b.s.valuesFor(e.Consumer, e.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issue 1482 (novox/hq ADR 0466 §3): the provider watches the build modules' state `keys` beside ssh-client's, and
|
||||
// registers each builder entry read-only on nox-builder.
|
||||
|
||||
func keyJSON(node, account, kind, public string) []byte {
|
||||
b, _ := json.Marshal(map[string]string{"node": node, "account": account, "kind": kind, "public": public,
|
||||
"fingerprint": "SHA256:x", "at": "2026-10-20T10:00:00Z"})
|
||||
return b
|
||||
}
|
||||
|
||||
// watched is a team rig whose keys come through watches of the modules' states, as in the running provider.
|
||||
type watched struct {
|
||||
*teamRig
|
||||
sources map[string]*fakeKeys
|
||||
passes *int
|
||||
logs *[]string
|
||||
mu *sync.Mutex
|
||||
}
|
||||
|
||||
func newWatched(t *testing.T) *watched {
|
||||
r := newTeamRig(t)
|
||||
r.svc.loaded, r.svc.loadedFrom = false, map[string]bool{}
|
||||
r.svc.Pause = func(time.Duration) {}
|
||||
n, logs, mu := 0, []string{}, &sync.Mutex{}
|
||||
r.svc.OnKeys = func(string) { n++ }
|
||||
r.svc.cfg.Log = func(f string, a ...any) { mu.Lock(); logs = append(logs, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
w := &watched{teamRig: r, sources: map[string]*fakeKeys{}, passes: &n, logs: &logs, mu: mu}
|
||||
for _, module := range []string{"ssh-client", "mesh-build-go", "mesh-build-node"} {
|
||||
w.sources[module] = &fakeKeys{}
|
||||
if err := r.svc.WatchSource(context.Background(), module, w.sources[module]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *watched) said() string {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return strings.Join(*w.logs, "\n")
|
||||
}
|
||||
|
||||
// 1482/AC1: a mesh-build-go.keys entry ace.mesh-build-go of kind builder: its public half is registered read-only on
|
||||
// nox-builder titled mesh:ace.mesh-build-go.
|
||||
func TestABuilderEntryOfABuildModulesStateIsRegisteredReadOnlyOnNoxBuilder(t *testing.T) {
|
||||
w := newWatched(t)
|
||||
w.bind("ace", "mesh-build-go")
|
||||
w.sources["mesh-build-go"].send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)"), true)
|
||||
w.pass()
|
||||
keys := w.forge.keys["nox-builder"]
|
||||
if len(keys) != 1 || keys[0].Title != "mesh:ace.mesh-build-go" || !keys[0].ReadOnly || keys[0].Key != pubGo+" mesh-build-go@ace (mesh)" {
|
||||
t.Errorf("nox-builder's keys = %+v", keys)
|
||||
}
|
||||
if !w.svc.loadedFrom["mesh-build-go"] {
|
||||
t.Errorf("the state was watched and its keys are not counted as read")
|
||||
}
|
||||
}
|
||||
|
||||
// 1482/AC2: that entry changed: the old key is removed and the new one registered; deleted: the key is removed and no key
|
||||
// titled otherwise is touched.
|
||||
func TestAChangedBuilderEntryReplacesItsKeyAndADeletedOneRemovesIt(t *testing.T) {
|
||||
w := newWatched(t)
|
||||
w.bind("ace", "mesh-build-go")
|
||||
src := w.sources["mesh-build-go"]
|
||||
src.send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "builder", pubGo), true)
|
||||
w.pass()
|
||||
w.forge.mu.Lock()
|
||||
w.forge.keys["nox-builder"] = append(w.forge.keys["nox-builder"], forgeKey{ID: 99, Title: "my own", Key: pubOps})
|
||||
w.forge.mu.Unlock()
|
||||
|
||||
before := *w.passes
|
||||
src.send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "builder", pubNode), false)
|
||||
if *w.passes != before+1 {
|
||||
t.Errorf("a change asked for %d passes, want 1", *w.passes-before)
|
||||
}
|
||||
w.pass()
|
||||
got := w.forge.keys["nox-builder"]
|
||||
if len(got) != 2 || got[0].Title != "my own" || got[1].Title != "mesh:ace.mesh-build-go" || got[1].Key != pubNode || !got[1].ReadOnly {
|
||||
t.Errorf("after the change: %+v", got)
|
||||
}
|
||||
|
||||
src.send("ace.mesh-build-go", "delete", nil, false)
|
||||
w.pass()
|
||||
if got := strings.Join(w.forge.titles("nox-builder"), ","); got != "my own" {
|
||||
t.Errorf("after the delete: %s, want only the operator's key", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1482/AC3: an entry of kind machine in a build module's state, or of kind builder in ssh-client's, is ignored and
|
||||
// said in the provider's log.
|
||||
func TestAKeyOfTheWrongKindForItsStateIsIgnoredAndSaid(t *testing.T) {
|
||||
w := newWatched(t)
|
||||
w.bind("ace", "mesh-build-go")
|
||||
w.bind("g14", "ssh-client")
|
||||
w.sources["mesh-build-go"].send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "machine", pubGo), true)
|
||||
w.sources["ssh-client"].send("g14.nox", "put", keyJSON("g14", "nox", "builder", pubA), true)
|
||||
w.pass()
|
||||
if len(w.forge.users) != 0 || w.bus.has("ace.mesh-build-go.forge-account") || w.bus.has("g14.ssh-client.forge-account") {
|
||||
t.Errorf("a key of the wrong kind was served: users %v", w.forge.users)
|
||||
}
|
||||
said := w.said()
|
||||
for _, want := range []string{`ace.mesh-build-go in the state keys of mesh-build-go is of kind "machine", not "builder"`, `g14.nox in the state keys of ssh-client is of kind "builder", not "machine"`} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Errorf("the log lacks %q:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A build module's state that never answers holds back its own entries, never a machine's.
|
||||
func TestABuildModulesStateThatNeverAnswersHoldsBackOnlyItsOwnEntries(t *testing.T) {
|
||||
w := newWatched(t)
|
||||
w.svc.loadedFrom = map[string]bool{} // mesh-build-go's watch has not answered
|
||||
w.bind("ace", "mesh-build-go")
|
||||
w.bind("g14", "ssh-client")
|
||||
w.sources["ssh-client"].send("g14.nox", "put", keyJSON("g14", "nox", "machine", pubA+" nox@g14 (mesh)"), true)
|
||||
out := w.pass()
|
||||
if !w.bus.has("g14.ssh-client.forge-account") {
|
||||
t.Errorf("a machine's entry waited for a build module's state")
|
||||
}
|
||||
waits := 0
|
||||
for _, o := range out {
|
||||
if strings.Contains(o.Key, "mesh-build-go") && strings.Contains(o.Why, "keys of mesh-build-go are not read yet") {
|
||||
waits++
|
||||
}
|
||||
}
|
||||
if waits != 1 {
|
||||
t.Errorf("outcomes = %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// 1482/AC4 (the provider's half): the manifest reads the states of both build modules beside ssh-client's and the
|
||||
// controller's assignments; the provider watches exactly the modules the code names as builders.
|
||||
func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Reads []string `json:"reads"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys"} {
|
||||
if !slices.Contains(m.Reads, state) {
|
||||
t.Errorf("gitea's reads %v lack %q", m.Reads, state)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
// Package forgeaccount is the forge accounts the mesh makes for its agents (novox/hq ADR 0365 clauses 1 and 3, ADR
|
||||
// 0398 §1, ADR 0399, ADR 0400; issue 955): an account of the agent's own, no administrator, in the agents write
|
||||
// team. Both the token tool (cmd/agent-token) and the forge-account provider use it, so there is one place that says
|
||||
// what an agent's account is.
|
||||
// what an agent's account is. It also makes the one account of the build modules' keys, nox-builder, in the builders
|
||||
// team that reads the code of every repository and nothing else (ADR 0466 §3; issue 1481).
|
||||
package forgeaccount
|
||||
|
||||
import (
|
||||
@@ -26,8 +27,23 @@ const (
|
||||
TeamName = "agents"
|
||||
)
|
||||
|
||||
// teamShape is what the agents team grants: write on code, issues and pull requests of every repository of the
|
||||
// BuildersTeamName is the team of the build modules' account (ADR 0466 §3).
|
||||
const BuildersTeamName = "builders"
|
||||
|
||||
// Team is a team of the organisation as the mesh keeps it: its name and the shape it is put back to.
|
||||
type Team struct {
|
||||
Name string
|
||||
Shape func() map[string]any
|
||||
}
|
||||
|
||||
// AgentsTeam is the team of the machines' agents: write on code, issues and pull requests of every repository of the
|
||||
// organisation and no more, the shape of the forge tools' team (tools-account.ts).
|
||||
var AgentsTeam = Team{Name: TeamName, Shape: teamShape}
|
||||
|
||||
// BuildersTeam is the team of the build modules' account: it may read the code of every repository of the organisation
|
||||
// and has no other right (ADR 0466 §3).
|
||||
var BuildersTeam = Team{Name: BuildersTeamName, Shape: buildersTeamShape}
|
||||
|
||||
func teamShape() map[string]any {
|
||||
return map[string]any{
|
||||
"name": TeamName, "permission": "write", "includes_all_repositories": true, "can_create_org_repo": false,
|
||||
@@ -35,6 +51,13 @@ func teamShape() map[string]any {
|
||||
}
|
||||
}
|
||||
|
||||
func buildersTeamShape() map[string]any {
|
||||
return map[string]any{
|
||||
"name": BuildersTeamName, "permission": "read", "includes_all_repositories": true, "can_create_org_repo": false,
|
||||
"units_map": map[string]string{"repo.code": "read"},
|
||||
}
|
||||
}
|
||||
|
||||
// User is gitea's account as the admin reads it.
|
||||
type User struct {
|
||||
IsAdmin bool `json:"is_admin"`
|
||||
@@ -87,35 +110,41 @@ func (m *Accounts) EnsureAccount(ctx context.Context, account string) error {
|
||||
// EnsureInTeam makes the agents team when there is none, puts a team that is there back to its shape, and adds the
|
||||
// account to it; an account already in it stays (ADR 0398 §1, issue 948).
|
||||
func (m *Accounts) EnsureInTeam(ctx context.Context, account string) error {
|
||||
id, err := m.teamID(ctx)
|
||||
return m.EnsureInTeamOf(ctx, AgentsTeam, account)
|
||||
}
|
||||
|
||||
// EnsureInTeamOf is EnsureInTeam for any team of the mesh's: it makes the team when there is none, puts a team that is
|
||||
// there back to its shape, and adds the account to it.
|
||||
func (m *Accounts) EnsureInTeamOf(ctx context.Context, team Team, account string) error {
|
||||
id, err := m.teamID(ctx, team.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if id != 0 {
|
||||
if err := m.Forge.SendJSON(ctx, http.MethodPatch, fmt.Sprintf("/api/v1/teams/%d", id), teamShape(), nil); err != nil {
|
||||
return fmt.Errorf("the team %s could not be put back to its shape: %w", TeamName, err)
|
||||
if err := m.Forge.SendJSON(ctx, http.MethodPatch, fmt.Sprintf("/api/v1/teams/%d", id), team.Shape(), nil); err != nil {
|
||||
return fmt.Errorf("the team %s could not be put back to its shape: %w", team.Name, err)
|
||||
}
|
||||
} else {
|
||||
var made struct {
|
||||
ID int64 `json:"id"`
|
||||
}
|
||||
if err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/orgs/"+TeamOrg+"/teams", teamShape(), &made); err != nil {
|
||||
return fmt.Errorf("the team %s could not be made: %w", TeamName, err)
|
||||
if err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/orgs/"+TeamOrg+"/teams", team.Shape(), &made); err != nil {
|
||||
return fmt.Errorf("the team %s could not be made: %w", team.Name, err)
|
||||
}
|
||||
id = made.ID
|
||||
m.say("made the forge team %s in %s: write on every repository", TeamName, TeamOrg)
|
||||
m.say("made the forge team %s in %s", team.Name, TeamOrg)
|
||||
}
|
||||
if id == 0 {
|
||||
return fmt.Errorf("the forge answered no id for the team %s", TeamName)
|
||||
return fmt.Errorf("the forge answered no id for the team %s", team.Name)
|
||||
}
|
||||
if err := m.Forge.SendJSON(ctx, http.MethodPut, fmt.Sprintf("/api/v1/teams/%d/members/%s", id, account), nil, nil); err != nil {
|
||||
return fmt.Errorf("the account %s could not be added to the team %s: %w", account, TeamName, err)
|
||||
return fmt.Errorf("the account %s could not be added to the team %s: %w", account, team.Name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// teamID is the agents team's id, or 0 when there is none.
|
||||
func (m *Accounts) teamID(ctx context.Context) (int64, error) {
|
||||
// teamID is the id of the team of that name, or 0 when there is none.
|
||||
func (m *Accounts) teamID(ctx context.Context, name string) (int64, error) {
|
||||
var id int64
|
||||
for page := 1; page <= 20 && id == 0; page++ {
|
||||
var teams []struct {
|
||||
@@ -126,7 +155,7 @@ func (m *Accounts) teamID(ctx context.Context) (int64, error) {
|
||||
return 0, fmt.Errorf("the teams of %s could not be read: %w", TeamOrg, err)
|
||||
}
|
||||
for _, t := range teams {
|
||||
if t.Name == TeamName {
|
||||
if t.Name == name {
|
||||
id = t.ID
|
||||
}
|
||||
}
|
||||
@@ -160,16 +189,49 @@ var NodeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`)
|
||||
// AgentUser is the one forge user of every machine's agents (ADR 0410 §1), in the agents write team.
|
||||
const AgentUser = "nox-agent"
|
||||
|
||||
// ForgeUser is the forge user of a machine's key of the given kind (ADR 0410 §1): `nox-agent` for every machine
|
||||
// and account (kind machine). A kind not built yet has none, and the answer is empty. The machine's name must still
|
||||
// be one an account could be named for.
|
||||
// BuilderUser is the one forge user of every build module's key (ADR 0466 §3), in the builders team: it reads and
|
||||
// pushes nothing.
|
||||
const BuilderUser = "nox-builder"
|
||||
|
||||
// BuildModules are the modules whose keys are the builders' (kind builder), by their name, which is also the name of
|
||||
// the account each runs as (ADR 0466 §1).
|
||||
var BuildModules = []string{"mesh-build-go", "mesh-build-node"}
|
||||
|
||||
// KindOfModule is the kind of key a consumer module's entry is made for: builder for a build module, machine for any
|
||||
// other (ssh-client's).
|
||||
func KindOfModule(module string) string {
|
||||
for _, m := range BuildModules {
|
||||
if m == module {
|
||||
return "builder"
|
||||
}
|
||||
}
|
||||
return "machine"
|
||||
}
|
||||
|
||||
// ForgeUser is the forge user of a key of the given kind (ADR 0410 §1, ADR 0466 §3): `nox-agent` for every machine
|
||||
// and account (kind machine), `nox-builder` for every build module (kind builder). A kind not built yet has none, and
|
||||
// the answer is empty. The machine's name must still be one an account could be named for.
|
||||
func ForgeUser(node, kind string) string {
|
||||
if kind == "machine" && NodeName.MatchString(node) {
|
||||
if !NodeName.MatchString(node) {
|
||||
return ""
|
||||
}
|
||||
switch kind {
|
||||
case "machine":
|
||||
return AgentUser
|
||||
case "builder":
|
||||
return BuilderUser
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// TeamOfUser is the team a forge user of the mesh is in.
|
||||
func TeamOfUser(user string) Team {
|
||||
if user == BuilderUser {
|
||||
return BuildersTeam
|
||||
}
|
||||
return AgentsTeam
|
||||
}
|
||||
|
||||
// NodeAccount is the forge account the token tool (cmd/agent-token) keeps for a node's agent: `agent-<node>` (ADR
|
||||
// 0365 §1). It is not the forge user of the machine's ssh key (ForgeUser), and its behaviour is unchanged.
|
||||
func NodeAccount(node string) string { return AccountPrefix + node }
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package forgeaccount
|
||||
|
||||
import "testing"
|
||||
|
||||
// Issue 1481 (novox/hq ADR 0466 §3): the build modules' forge user, team and key scope.
|
||||
func TestTheBuildModulesHaveOneForgeUserAndTheMachinesKeepTheirs(t *testing.T) {
|
||||
for kind, want := range map[string]string{"machine": "nox-agent", "builder": "nox-builder", "session": ""} {
|
||||
if got := ForgeUser("ace", kind); got != want {
|
||||
t.Errorf("ForgeUser(ace, %s) = %q, want %q", kind, got, want)
|
||||
}
|
||||
}
|
||||
if ForgeUser("Not A Node", "builder") != "" {
|
||||
t.Errorf("a name no account could be named for got a user")
|
||||
}
|
||||
for module, want := range map[string]string{"mesh-build-go": "builder", "mesh-build-node": "builder", "ssh-client": "machine", "": "machine"} {
|
||||
if got := KindOfModule(module); got != want {
|
||||
t.Errorf("KindOfModule(%q) = %s, want %s", module, got, want)
|
||||
}
|
||||
}
|
||||
if TeamOfUser(BuilderUser).Name != "builders" || TeamOfUser(AgentUser).Name != "agents" {
|
||||
t.Errorf("teams: %s and %s", TeamOfUser(BuilderUser).Name, TeamOfUser(AgentUser).Name)
|
||||
}
|
||||
}
|
||||
|
||||
// A build module's scope names the module, so one module's pass does not remove the other's key on the same machine;
|
||||
// the machine's scope is every key of the machine.
|
||||
func TestABuilderScopeNamesTheModule(t *testing.T) {
|
||||
if got := BuilderScope("ace", "mesh-build-go"); got != "mesh:ace.mesh-build-go" {
|
||||
t.Errorf("scope = %q", got)
|
||||
}
|
||||
if BuilderScope("ace", "mesh-build-go") == BuilderScope("ace", "mesh-build-node") {
|
||||
t.Errorf("two build modules share a scope")
|
||||
}
|
||||
if got := KeyTitle("ace", "mesh-build-go"); got != BuilderScope("ace", "mesh-build-go") {
|
||||
t.Errorf("the key's title %q is not inside the scope", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The builders team reads the code of every repository of the organisation and has no other right.
|
||||
func TestTheBuildersTeamReadsCodeOnly(t *testing.T) {
|
||||
s := BuildersTeam.Shape()
|
||||
units, _ := s["units_map"].(map[string]string)
|
||||
if s["permission"] != "read" || s["includes_all_repositories"] != true || s["can_create_org_repo"] != false || len(units) != 1 || units["repo.code"] != "read" {
|
||||
t.Errorf("shape = %v", s)
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,8 @@ type WantedKey struct {
|
||||
Title string
|
||||
// Key is the public key line: type, base64 and a comment.
|
||||
Key string
|
||||
// ReadOnly registers the key as one that may not push (a builder's, ADR 0466 §3).
|
||||
ReadOnly bool
|
||||
}
|
||||
|
||||
// Key is a key the forge holds for an account.
|
||||
@@ -56,10 +58,10 @@ func (m *Accounts) Keys(ctx context.Context, user string) ([]Key, error) {
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// AddKey registers a key for the account, one it may write with.
|
||||
// AddKey registers a key for the account, one it may write with unless the key is wanted read-only.
|
||||
func (m *Accounts) AddKey(ctx context.Context, user string, k WantedKey) error {
|
||||
err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/admin/users/"+user+"/keys",
|
||||
map[string]any{"title": k.Title, "key": k.Key, "read_only": false}, nil)
|
||||
map[string]any{"title": k.Title, "key": k.Key, "read_only": k.ReadOnly}, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the key %s could not be added for %s: %w", k.Title, user, err)
|
||||
}
|
||||
@@ -78,6 +80,11 @@ func (m *Accounts) DeleteKey(ctx context.Context, user string, id int64) error {
|
||||
// MachineScope is the title prefix of one machine's keys, `mesh:<node>.`: what a pass for that machine may remove.
|
||||
func MachineScope(node string) string { return TitlePrefix + node + "." }
|
||||
|
||||
// BuilderScope is the title prefix of one build module's key on one machine, `mesh:<node>.<module>`: what a pass for
|
||||
// that module's entry may remove. It names the module, since nox-builder holds the keys of both build modules of a
|
||||
// machine and one's pass must not remove the other's.
|
||||
func BuilderScope(node, module string) string { return TitlePrefix + node + "." + module }
|
||||
|
||||
// EnsureKeys registers each wanted key the account does not hold and removes the keys the mesh registered
|
||||
// for one machine (title starting scope, `mesh:<node>.`) that are no longer wanted. One forge user holds every
|
||||
// machine's keys (ADR 0410 §2), so a key under another machine's title is never removed here. A key whose title does not start `mesh:` is the operator's and
|
||||
|
||||
@@ -540,6 +540,8 @@
|
||||
],
|
||||
"reads": [
|
||||
"mesh-controller.assignments",
|
||||
"ssh-client.keys"
|
||||
"ssh-client.keys",
|
||||
"mesh-build-go.keys",
|
||||
"mesh-build-node.keys"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# mesh-build-go — the Go build module
|
||||
|
||||
Design 47, release R4c-3 (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470). The module that builds and checks Go
|
||||
code on a build machine, as an account of its own (`runs-as`, no login shell, home at mode 0700), with the
|
||||
toolchain installed as the packages it declares (`go`, `git`, `openssh`, `postgresql`) and the capability
|
||||
`user-namespaces`. It claims no seat: it is assigned to the build machines (ADR 0469 §6) and runs dark until the
|
||||
builder uses it.
|
||||
|
||||
## The recipe
|
||||
|
||||
`cmd/mesh-build-go/recipe.go` names the one Go version the module builds with (`RecipeGo`), the postgres major
|
||||
version the integration level tests with, and the recipe's own version. The recipe version is a hash of those and of
|
||||
the nats-server library's version, read from the program's build information, so a change of any of them moves it.
|
||||
|
||||
## The guard and the health
|
||||
|
||||
At start, every ten minutes, before every step and at every health look, the module reads the installed Go
|
||||
(`go version`) and writes it with the recipe version to `<node>.go` of its `toolchains` state. A machine whose Go
|
||||
is not the recipe's refuses every step, the step ending `refused` on its key in `steps` with both versions named,
|
||||
and `build_go_health` (the module's `runtime-health`) answers unhealthy with the same text, so the controller raises
|
||||
the machine's condition. Moving to a new Go is a change of `RecipeGo`.
|
||||
|
||||
## Tools
|
||||
|
||||
`build_go_status`, `build_go_health` and `build_go_fetch` are served now; the other step tools (`build_go_test`,
|
||||
`build_go_digest`, `build_go_build`) are added by their own issues to `StepTools` and run behind the guard: a long
|
||||
step answers at once with its id and ends on `<node>.<id>` of `steps` (ADR 0469 §2); the digest answers in its call.
|
||||
|
||||
## Fetch
|
||||
|
||||
`build_go_fetch {repository, commit, ref?}` fetches the commit over SSH as the module's account (`mesh-forge` is the
|
||||
forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the
|
||||
work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor,
|
||||
with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an
|
||||
environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day
|
||||
are removed.
|
||||
|
||||
## Test
|
||||
|
||||
`build_go_test {workspace, level, packages?}` runs the tests of a fetched workspace by Go's convention, one level at a
|
||||
time. The unit level is `go test -json` over every package without the `integration` build tag, in each directory
|
||||
with a `go.mod`, judged by its one bound: its running time, that of the slowest package (compile not counted), within
|
||||
five seconds. The step ends passed when the level ran, and its result says whether the level did: `passed`, the
|
||||
packages with their times, the tests that failed with their first lines of output, the five slowest tests, and `why`
|
||||
when it failed (a failed test, a build that failed) or ran over a bound. `passed` is the tests' own verdict: a level over its bound still passes and says `over_bound` (and `over_which`, setup or run) with the slowest tests, and the checker judges the bounds (ADR 0453 §9). A step ends failed only when go test could not
|
||||
run. A repository with no tests at the level has none, and the result says so: that is a pass. `testdata/captured` holds
|
||||
the output of the real go for the fixture repositories in `testdata/repos`; the unit tests read it, and the
|
||||
integration-tagged test runs the real go on the fixtures.
|
||||
|
||||
## The integration level
|
||||
|
||||
`build_go_test` with `level: integration` runs `go test -tags integration` against a store and a bus started for the
|
||||
run alone (novox/hq ADR 0470). The store is a postgres of the installed package: `initdb` and `pg_ctl` into
|
||||
`<work>/<workspace>/run-<id>/pg`, as the module's account, listening on a unix socket there and on no TCP address, with
|
||||
durability off; the tests get `MESH_TEST_POSTGRES` (the socket directory percent-encoded in the URL, because the
|
||||
controller's tests cut the URL at its last slash). The bus is a nats-server from the Go library at the version this
|
||||
module's `go.mod` names, run by this same binary started as `mesh-build-go bus <dir>` (`bus.go`: JetStream on, a
|
||||
loopback port of its own, its address on the first line, stopped when its input closes); the tests get `MESH_TEST_NATS`.
|
||||
Both run inside the step, are stopped, and the run's directory is removed whatever the tests did.
|
||||
|
||||
Setup (compiling the tests, the store, the bus) is judged within 30 s and the tests' running time (the wall time of
|
||||
`go test`) within 60 s, apart; a level over either says `over_bound` and names the bound, with the slowest tests for the running
|
||||
time. A machine whose installed postgres major is not the recipe's refuses this level, naming both; the unit level
|
||||
still runs. The result names the postgres major and the nats-server version it ran against. The mesh's own store major
|
||||
is not named: no state or grant gives it to this module (ADR 0470 §4 asks for it when it differs), so only the recipe's
|
||||
is said until a source exists.
|
||||
|
||||
## Digest
|
||||
|
||||
`build_go_digest {workspace, level}` answers in its call, for each package of the workspace that has tests at the level,
|
||||
one digest of everything those tests read (ADR 0445 §5): the files, by content and under their path in the tree, of the
|
||||
package and every package its test binary reaches (`go list -test -deps`) that lies in the module's own tree; the version
|
||||
and checksum of every module outside it; the package's own `testdata/` files; and the Go version, the level, the build tags
|
||||
and, at the integration level, the postgres major and the nats-server version. Packages of the standard library are the Go
|
||||
version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors
|
||||
is listed and tested from its vendor directory (`-mod=vendor`).
|
||||
|
||||
## Build
|
||||
|
||||
`build_go_build {workspace, module, artifact}` builds one Go bundle artifact of a module of the workspace with the build
|
||||
agent's recipe (mesh-controller's `toolchain.go`, the Go entry, copied in `testdata/captured/toolchain-go-entry.txt`, and
|
||||
`builder.go`): `CGO_ENABLED=0`, `GOFLAGS=-trimpath -buildvcs=false` (with `-mod=mod`, or `-mod=vendor` for a directory
|
||||
that vendors), `go build -ldflags "-s -w -X main.builtFor=<system>" -o .mesh-build/<artifact>/<binary> ./<from>` in the
|
||||
module's directory, by the installed Go (`GOTOOLCHAIN=local`) from the module cache the fetch filled and no network
|
||||
(`GOPROXY=off`). The binary is named as the build agent names it. The output directory is packed with mesh-sdk's `Pack`,
|
||||
so the digest is the build agent's for the same tree. The step ends with the artifact, the binary, the digest, the size and
|
||||
the path of the archive, which stays in the workspace's `artifacts` directory: publishing it is the builder's, in a later
|
||||
release. An artifact that is not a Go bundle fails the step naming it.
|
||||
@@ -0,0 +1,164 @@
|
||||
package main
|
||||
|
||||
// The build step (novox/hq ADR 0405 §3 §7, ADR 0453 §7; issue 1487): a Go bundle artifact of a module of a fetched
|
||||
// workspace, built with today's recipe — the build agent's, mesh-controller's internal/builder/toolchain.go (the Go
|
||||
// entry) and builder.go (compile, binaryName, pack) — by the installed Go and no network, packed with mesh-sdk's Pack
|
||||
// so that its digest is the build agent's for the same tree. The archive is left in the workspace; publishing it is
|
||||
// the builder's, in a later release.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The Go recipe of the build agent (toolchain.go, the entry of language go): static, trimmed of its own paths and of
|
||||
// the commit stamp, stripped of symbols and debug info, and told the system it was built for.
|
||||
const (
|
||||
recipeFlags = "-trimpath -buildvcs=false"
|
||||
recipeLinkerFlags = "-s -w"
|
||||
recipeSystemStamp = "main.builtFor"
|
||||
// outDir is where a build leaves an artifact in the module's directory (builder.Out).
|
||||
outDir = ".mesh-build"
|
||||
)
|
||||
|
||||
// BuildResult is what a passed build step says.
|
||||
type BuildResult struct {
|
||||
Artifact string `json:"artifact"`
|
||||
Binary string `json:"binary"`
|
||||
Digest string `json:"digest"`
|
||||
Size int `json:"size"`
|
||||
// Path is the archive, in the workspace's directory of the work directory.
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_go_build",
|
||||
Description: "Build one Go bundle artifact of a module of a fetched workspace with the build agent's recipe, by the " +
|
||||
"installed Go and no network, and pack it. Answers a step id at once; the step ends on its key with the " +
|
||||
"artifact's digest, size and the archive's path.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"},
|
||||
},
|
||||
"required": []string{"workspace", "module", "artifact"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.build,
|
||||
}
|
||||
}
|
||||
|
||||
// binaryName is what a compiled bundle's executable is called (builder.go binaryName): what the artifact says, or the
|
||||
// name of the package it is built from, or the artifact's.
|
||||
func binaryName(a catalogue.Artifact) string {
|
||||
if name := catalogue.BinaryOf(a); name != "" {
|
||||
return name
|
||||
}
|
||||
if name := strings.TrimSpace(a.Binary); name != "" {
|
||||
return name
|
||||
}
|
||||
if from := strings.Trim(a.From, "./"); from != "" {
|
||||
return filepath.Base(from)
|
||||
}
|
||||
return a.Name
|
||||
}
|
||||
|
||||
func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
tree := filepath.Join(m.Work, workspace, "tree")
|
||||
dir := filepath.Join(tree, filepath.FromSlash(module))
|
||||
if rel, err := filepath.Rel(tree, dir); err != nil || strings.HasPrefix(rel, "..") {
|
||||
return nil, fmt.Errorf("the module directory %q lies outside the workspace", module)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifest in %q: %w", module, err)
|
||||
}
|
||||
var art *catalogue.Artifact
|
||||
names := []string{}
|
||||
if manifest.Build != nil {
|
||||
for i := range manifest.Build.Artifacts {
|
||||
names = append(names, manifest.Build.Artifacts[i].Name)
|
||||
if manifest.Build.Artifacts[i].Name == name {
|
||||
art = &manifest.Build.Artifacts[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if art == nil {
|
||||
return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", "))
|
||||
}
|
||||
if art.Kind != "bundle" || art.Language != "go" {
|
||||
return nil, fmt.Errorf("the artifact %s of %s is a %s of language %q: this module builds Go bundles", name, manifest.Module, art.Kind, art.Language)
|
||||
}
|
||||
if strings.TrimSpace(art.From) == "" {
|
||||
return nil, fmt.Errorf("the artifact %s of %s says nothing it is built from", name, manifest.Module)
|
||||
}
|
||||
|
||||
// Its own output directory, made empty: a build of the same tree twice is one build.
|
||||
out := filepath.Join(dir, outDir, art.Name)
|
||||
if err := os.RemoveAll(out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.MkdirAll(out, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("making the output directory for %s: %w", art.Name, err)
|
||||
}
|
||||
binary := binaryName(*art)
|
||||
// One -ldflags, composed: a repeated flag is not a merged one (the build agent's issue 161).
|
||||
linker := recipeLinkerFlags
|
||||
if system := strings.TrimSpace(art.System); system != "" {
|
||||
linker += " -X " + recipeSystemStamp + "=" + system
|
||||
}
|
||||
cmd := []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(art.From, "./")}
|
||||
if r := m.Run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("go build of %s: %s", art.Name, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
archive, digest, err := buildstep.Pack(out)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("packing %s: %w", art.Name, err)
|
||||
}
|
||||
artifacts := filepath.Join(m.Work, workspace, "artifacts")
|
||||
if err := os.MkdirAll(artifacts, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path := filepath.Join(artifacts, strings.ReplaceAll(module, "/", "_")+"."+art.Name+".tar.gz")
|
||||
if err := os.WriteFile(path, archive, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return BuildResult{Artifact: art.Name, Binary: binary, Digest: digest, Size: len(archive), Path: path}, nil
|
||||
}
|
||||
|
||||
// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed toolchain
|
||||
// alone, the modules the fetch put in the module's cache and no way to fetch another (GOPROXY=off), so a build never
|
||||
// reaches the network. A directory that vendors is built from its vendor directory.
|
||||
func (m *Module) buildEnv(dir string) []string {
|
||||
mod := "-mod=mod"
|
||||
if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
|
||||
mod = "-mod=vendor"
|
||||
}
|
||||
return []string{
|
||||
"CGO_ENABLED=0",
|
||||
"GOFLAGS=" + recipeFlags + " " + mod,
|
||||
"GOTOOLCHAIN=local",
|
||||
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
|
||||
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
|
||||
"GOPROXY=off",
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Issue 1487: the build step, with the compiler faked at the process boundary: it writes the file `go build -o` names.
|
||||
|
||||
func manifestWith(artifact string) string {
|
||||
return `{"module":"demo","version":"1","build":{"artifacts":[` + artifact + `]}}`
|
||||
}
|
||||
|
||||
const goBundle = `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","binary":"demo-bin","loads":["demo-bin"]}`
|
||||
|
||||
// buildModule is a module with a workspace holding a module "modules/demo" with this manifest; the compiler writes
|
||||
// content to the file go build is told.
|
||||
func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates, *[]cmdRecord) {
|
||||
t.Helper()
|
||||
calls := &[]cmdRecord{}
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
|
||||
dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
*calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")})
|
||||
for i, a := range args {
|
||||
if a == "-o" {
|
||||
if err := os.WriteFile(args[i+1], []byte(content), 0o755); err != nil {
|
||||
return forgekey.Ran{Status: 1, Stderr: err.Error()}
|
||||
}
|
||||
}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
return m, states, calls
|
||||
}
|
||||
|
||||
func runBuild(t *testing.T, m *Module, states *keptStates, artifact string) (BuildResult, delivery.Step) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == "build_go_build" {
|
||||
if _, err := tool.Run(map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": artifact}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res BuildResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
// goEntry reads what the build agent's recipe says out of lines 155-191 of mesh-control's internal/builder/toolchain.go
|
||||
// (testdata/captured/toolchain-go-entry.txt, copied from origin/main 073ccc29): the environment words and flags of the
|
||||
// compile, the linker flags and the system stamp.
|
||||
func goEntry(t *testing.T) (compile []string, linker []string, stamp string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "captured", "toolchain-go-entry.txt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(raw)
|
||||
find := func(pattern string) string {
|
||||
m := regexp.MustCompile(pattern).FindStringSubmatch(text)
|
||||
if m == nil {
|
||||
t.Fatalf("the copied recipe has nothing like %s", pattern)
|
||||
}
|
||||
return m[1]
|
||||
}
|
||||
for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`Compile: \[\]string\{([^}]*)\}`), -1) {
|
||||
compile = append(compile, q[1])
|
||||
}
|
||||
for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`LinkerFlags: \[\]string\{([^}]*)\}`), -1) {
|
||||
linker = append(linker, q[1])
|
||||
}
|
||||
return compile, linker, find(`SystemStamp: "([^"]+)"`)
|
||||
}
|
||||
|
||||
// 1487/AC1: the build command carries today's flags and environment: the copied recipe says env CGO_ENABLED=0
|
||||
// GOFLAGS=-trimpath -buildvcs=false, go build, -ldflags -s -w -X main.builtFor=<system>, -o <out>/<binary>, the package.
|
||||
func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) {
|
||||
m, states, calls := buildModule(t, manifestWith(goBundle), "binary")
|
||||
res, st := runBuild(t, m, states, "tools")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
compile, linker, stamp := goEntry(t)
|
||||
// compile is: env CGO_ENABLED=0 "GOFLAGS=-trimpath -buildvcs=false" go build
|
||||
if len(compile) != 5 || compile[0] != "env" || compile[3] != "go" || compile[4] != "build" {
|
||||
t.Fatalf("the copied recipe's compile is %v", compile)
|
||||
}
|
||||
var c cmdRecord
|
||||
for _, r := range *calls {
|
||||
if strings.HasPrefix(r.line, "go ") {
|
||||
c = r
|
||||
}
|
||||
}
|
||||
env := strings.Join(c.env, "\n")
|
||||
for _, want := range []string{compile[1], "GOFLAGS=" + strings.TrimPrefix(compile[2], "GOFLAGS=") + " -mod=mod", "GOTOOLCHAIN=local", "GOPROXY=off",
|
||||
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild")} {
|
||||
if !strings.Contains(env, want) {
|
||||
t.Errorf("the environment lacks %q:\n%s", want, env)
|
||||
}
|
||||
}
|
||||
dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
|
||||
out := filepath.Join(dir, ".mesh-build", "tools")
|
||||
want := "go -C " + dir + " build -ldflags " + strings.Join(linker, " ") + " -X " + stamp + "=arch -o " + filepath.Join(out, "demo-bin") + " ./cmd/demo-tools"
|
||||
if c.line != want {
|
||||
t.Errorf("command = %q\nwant %q", c.line, want)
|
||||
}
|
||||
if res.Artifact != "tools" || res.Binary != "demo-bin" || !strings.HasPrefix(res.Digest, "sha256:") || res.Size == 0 {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// The archive is the file the digest names, holding one executable of the name binaryName says; two builds of one tree
|
||||
// are one digest.
|
||||
func TestTheArchiveHoldsOneExecutableAndTwoBuildsAreOneDigest(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
first, _ := runBuild(t, m, states, "tools")
|
||||
second, _ := runBuild(t, m, states, "tools")
|
||||
if first.Digest != second.Digest || first.Path != second.Path {
|
||||
t.Errorf("two builds: %+v and %+v", first, second)
|
||||
}
|
||||
raw, err := os.ReadFile(first.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
if first.Digest != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
t.Errorf("the digest %s is not the archive's", first.Digest)
|
||||
}
|
||||
zr, err := gzip.NewReader(strings.NewReader(string(raw)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tr := tar.NewReader(zr)
|
||||
h, err := tr.Next()
|
||||
if err != nil || h.Name != "demo-bin" || h.Mode != 0o755 {
|
||||
t.Fatalf("first entry = %+v, %v", h, err)
|
||||
}
|
||||
if body, _ := io.ReadAll(tr); string(body) != "the program" {
|
||||
t.Errorf("the executable holds %q", body)
|
||||
}
|
||||
if _, err := tr.Next(); err != io.EOF {
|
||||
t.Errorf("the archive holds more than one file")
|
||||
}
|
||||
}
|
||||
|
||||
// 1487/AC3: an artifact of another language, or another kind, fails the step naming it.
|
||||
func TestAnArtifactOfAnotherLanguageFailsTheStepNamingIt(t *testing.T) {
|
||||
for artifact, want := range map[string]string{
|
||||
`{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"],"loads":["index.js"]}`: `language "typescript"`,
|
||||
`{"name":"tools","kind":"archive","from":"files"}`: "archive",
|
||||
} {
|
||||
m, states, calls := buildModule(t, manifestWith(artifact), "x")
|
||||
_, st := runBuild(t, m, states, "tools")
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, want) || len(*calls) != 0 {
|
||||
t.Errorf("%s: the step ended %s: %q after %d commands", artifact, st.State, st.Reason, len(*calls))
|
||||
}
|
||||
}
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "x")
|
||||
if _, st := runBuild(t, m, states, "nope"); st.State != delivery.StepFailed || !strings.Contains(st.Reason, `"nope"`) || !strings.Contains(st.Reason, "tools") {
|
||||
t.Errorf("an unknown artifact: %s %q", st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// The executable is named as the build agent names it: the artifact's binary, else the package's directory, else the
|
||||
// artifact.
|
||||
func TestTheBinaryIsNamedAsTheBuildAgentNamesIt(t *testing.T) {
|
||||
for artifact, want := range map[string]string{
|
||||
goBundle: "demo-bin",
|
||||
`{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","loads":["demo-tools"]}`: "demo-tools",
|
||||
`{"name":"tools","kind":"bundle","language":"go","system":"arch","from":".","loads":["tools"]}`: "tools",
|
||||
} {
|
||||
m, states, _ := buildModule(t, manifestWith(artifact), "x")
|
||||
if res, st := runBuild(t, m, states, "tools"); res.Binary != want {
|
||||
t.Errorf("%s: binary %q, want %q (%s %s)", artifact, res.Binary, want, st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleDirectoryOutsideTheWorkspaceIsRefused(t *testing.T) {
|
||||
m, states, calls := buildModule(t, manifestWith(goBundle), "x")
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == "build_go_build" {
|
||||
tool.Run(map[string]any{"workspace": "ws1", "module": "../../..", "artifact": "tools"})
|
||||
}
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepFailed || len(*calls) != 0 {
|
||||
t.Errorf("the step ended %s after %d commands", st.State, len(*calls))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package main
|
||||
|
||||
// The integration level's throwaway bus (novox/hq ADR 0470 §2; issue 1486): the module's own program runs a
|
||||
// nats-server from the Go library, at the one version this module's go.mod names, with JetStream on, its store in the
|
||||
// run's directory and a loopback port of its own. The program is this binary started again as `mesh-build-go bus
|
||||
// <dir>`, so the bus is built from the same go.mod as the version the recipe names (ADR 0470 §4) and no second program
|
||||
// has to be built and shipped; the tools bundle never serves a bus, and a bus never serves tools. It prints its
|
||||
// address on one line and stops when its standard input closes.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
)
|
||||
|
||||
// busStartBound is how long a bus is given to answer its address.
|
||||
const busStartBound = 30 * time.Second
|
||||
|
||||
// busArg is the first argument that makes this binary a bus.
|
||||
const busArg = "bus"
|
||||
|
||||
// runBus is the bus program: args are the directory of its store. It returns the exit status.
|
||||
func runBus(args []string) int {
|
||||
if len(args) != 1 {
|
||||
fmt.Fprintln(os.Stderr, "usage: mesh-build-go bus <store directory>")
|
||||
return 2
|
||||
}
|
||||
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: args[0], NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(busStartBound) {
|
||||
fmt.Fprintln(os.Stderr, "the bus was not ready within", busStartBound)
|
||||
s.Shutdown()
|
||||
return 1
|
||||
}
|
||||
fmt.Println(s.ClientURL())
|
||||
done := make(chan struct{})
|
||||
go func() { _, _ = io.Copy(io.Discard, os.Stdin); close(done) }()
|
||||
sig := make(chan os.Signal, 1)
|
||||
signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT)
|
||||
select {
|
||||
case <-done:
|
||||
case <-sig:
|
||||
}
|
||||
s.Shutdown()
|
||||
s.WaitForShutdown()
|
||||
return 0
|
||||
}
|
||||
|
||||
// startBus starts the bus of a run as a process of this binary under the step's environment, and answers its address.
|
||||
func (m *Module) startBus(ctx context.Context, dir string) (string, func(), error) {
|
||||
if m.StartBus != nil {
|
||||
return m.StartBus(ctx, dir)
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
cmd := exec.Command(exe, busArg, dir)
|
||||
cmd.Env = baseEnv(m.Home)
|
||||
in, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
out, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var errb strings.Builder
|
||||
cmd.Stderr = &errb
|
||||
if err := cmd.Start(); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
stop := func() {
|
||||
_ = in.Close()
|
||||
exited := make(chan struct{})
|
||||
go func() { _ = cmd.Wait(); close(exited) }()
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(10 * time.Second):
|
||||
_ = cmd.Process.Kill()
|
||||
<-exited
|
||||
}
|
||||
}
|
||||
line := make(chan string, 1)
|
||||
go func() {
|
||||
l, _ := bufio.NewReader(out).ReadString('\n')
|
||||
line <- strings.TrimSpace(l)
|
||||
}()
|
||||
select {
|
||||
case url := <-line:
|
||||
if !strings.HasPrefix(url, "nats://") {
|
||||
stop()
|
||||
return "", nil, fmt.Errorf("the bus did not answer an address: %s", firstLine(errb.String()+" "+url))
|
||||
}
|
||||
return url, stop, nil
|
||||
case <-time.After(busStartBound + 5*time.Second):
|
||||
stop()
|
||||
return "", nil, fmt.Errorf("the bus did not answer an address within %s", busStartBound)
|
||||
case <-ctx.Done():
|
||||
stop()
|
||||
return "", nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package main
|
||||
|
||||
// The digest step (novox/hq ADR 0445 §5, ADR 0444; issue 1291): for each package of a workspace that has tests at a
|
||||
// level, one digest of everything those tests read. The checker keys the verdict it keeps by it, so a package whose
|
||||
// digest is unchanged is not run again. The tool answers in its call (it is short, ADR 0469 §2).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
|
||||
const DigestVersion = "1"
|
||||
|
||||
// goPackage is the part of `go list -json` a digest reads.
|
||||
type goPackage struct {
|
||||
ImportPath string
|
||||
Dir string
|
||||
ForTest string
|
||||
Standard bool
|
||||
DepOnly bool
|
||||
Deps []string
|
||||
Module *goModule
|
||||
// The files a package is built and tested from.
|
||||
GoFiles, CgoFiles, CFiles, CXXFiles, MFiles, HFiles, FFiles, SFiles, SwigFiles, SwigCXXFiles, SysoFiles []string
|
||||
TestGoFiles, XTestGoFiles []string
|
||||
EmbedFiles, TestEmbedFiles, XTestEmbedFiles []string
|
||||
}
|
||||
|
||||
type goModule struct {
|
||||
Path, Version, Sum string
|
||||
Main bool
|
||||
Replace *goModule
|
||||
Dir string
|
||||
GoVersion string
|
||||
}
|
||||
|
||||
func (p *goPackage) files() []string {
|
||||
var all []string
|
||||
for _, l := range [][]string{p.GoFiles, p.CgoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles,
|
||||
p.SwigCXXFiles, p.SysoFiles, p.TestGoFiles, p.XTestGoFiles, p.EmbedFiles, p.TestEmbedFiles, p.XTestEmbedFiles} {
|
||||
all = append(all, l...)
|
||||
}
|
||||
sort.Strings(all)
|
||||
return slicesCompact(all)
|
||||
}
|
||||
|
||||
func slicesCompact(s []string) []string {
|
||||
out := s[:0:0]
|
||||
for i, v := range s {
|
||||
if i == 0 || v != s[i-1] {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hasTests says whether the package has test files at the level go list was asked for.
|
||||
func (p *goPackage) hasTests() bool { return len(p.TestGoFiles)+len(p.XTestGoFiles) > 0 }
|
||||
|
||||
// DigestResult is what a digest step answers: one digest per package that has tests at the level, and the packages
|
||||
// that have none.
|
||||
type DigestResult struct {
|
||||
Level string `json:"level"`
|
||||
Packages map[string]string `json:"packages"`
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Module) digestTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_go_digest",
|
||||
Description: "One digest per package of a fetched workspace, for one level, of everything that package's tests read: " +
|
||||
"the files of each package the tests reach in the module's own tree, the version and checksum of each module " +
|
||||
"outside it, the package's test data and embedded files, the Go version, the build tags and, at the " +
|
||||
"integration level, the postgres major and nats-server versions. Answers in the call.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
|
||||
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
|
||||
},
|
||||
"required": []string{"workspace", "level"},
|
||||
},
|
||||
Run: m.digest,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
level, _ := args["level"].(string)
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
tree := filepath.Join(m.Work, workspace, "tree")
|
||||
if _, err := os.Stat(tree); err != nil {
|
||||
return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
|
||||
}
|
||||
var tags []string
|
||||
header := []string{"digest " + DigestVersion, "level " + level, "go " + m.Recipe.Go}
|
||||
switch level {
|
||||
case LevelUnit:
|
||||
case LevelIntegration:
|
||||
if _, err := m.checkPostgres(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tags = []string{"integration"}
|
||||
header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer)
|
||||
default:
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
}
|
||||
header = append(header, "tags "+strings.Join(tags, ","))
|
||||
|
||||
dirs, err := m.goModDirs(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := DigestResult{Level: level, Packages: map[string]string{}}
|
||||
for _, d := range dirs {
|
||||
pkgs, err := m.listPackages(ctx, tree, d, tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := digestModule(&res, pkgs, tree, header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sort.Strings(res.WithoutTests)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// listPackages is `go list -test -deps -json` of the directory's packages, by import path.
|
||||
func (m *Module) listPackages(ctx context.Context, tree, dir string, tags []string) (map[string]*goPackage, error) {
|
||||
abs := filepath.Join(tree, dir)
|
||||
args := []string{"-C", abs, "list", "-test", "-deps", "-json"}
|
||||
if len(tags) > 0 {
|
||||
args = append(args, "-tags", strings.Join(tags, ","))
|
||||
}
|
||||
args = append(args, "./...")
|
||||
r := m.Run(ctx, m.goEnvFor(abs), "go", args...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("go list in %s: %s", dir, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
return parsePackages(r.Stdout)
|
||||
}
|
||||
|
||||
// parsePackages reads `go list -json`'s stream of objects.
|
||||
func parsePackages(out string) (map[string]*goPackage, error) {
|
||||
pkgs := map[string]*goPackage{}
|
||||
dec := json.NewDecoder(strings.NewReader(out))
|
||||
for {
|
||||
var p goPackage
|
||||
if err := dec.Decode(&p); err == io.EOF {
|
||||
return pkgs, nil
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("go list answered something that is not its JSON: %w", err)
|
||||
}
|
||||
pkgs[p.ImportPath] = &p
|
||||
}
|
||||
}
|
||||
|
||||
// digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main
|
||||
// module that is listed, not only a dependency, and not a variant of another.
|
||||
func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header []string) error {
|
||||
roots := []string{}
|
||||
for path, p := range pkgs {
|
||||
if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main {
|
||||
roots = append(roots, path)
|
||||
}
|
||||
}
|
||||
sort.Strings(roots)
|
||||
for _, path := range roots {
|
||||
root := pkgs[path]
|
||||
if !root.hasTests() {
|
||||
res.WithoutTests = append(res.WithoutTests, path)
|
||||
continue
|
||||
}
|
||||
testMain := pkgs[path+".test"]
|
||||
if testMain == nil {
|
||||
return fmt.Errorf("go list gave no test binary for %s", path)
|
||||
}
|
||||
d, err := packageDigest(root, testMain, pkgs, tree, header)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
res.Packages[path] = d
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// packageDigest is the digest of one package: a hash over a sorted text of the header, the files of every package its
|
||||
// tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of
|
||||
// every module outside it, and the package's own test data. A package of the standard library is the Go version's.
|
||||
func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header []string) (string, error) {
|
||||
reached := map[string]*goPackage{root.ImportPath: root}
|
||||
for _, dep := range testMain.Deps {
|
||||
if p := pkgs[dep]; p != nil {
|
||||
reached[dep] = p
|
||||
}
|
||||
}
|
||||
lines := []string{}
|
||||
for _, p := range reached {
|
||||
if p.Standard || p.Module == nil {
|
||||
continue
|
||||
}
|
||||
mod := p.Module
|
||||
if mod.Replace != nil {
|
||||
mod = mod.Replace
|
||||
}
|
||||
if !mod.Main && mod.Version != "" {
|
||||
lines = append(lines, "module "+mod.Path+"@"+mod.Version+" "+mod.Sum)
|
||||
continue
|
||||
}
|
||||
for _, f := range p.files() {
|
||||
l, err := fileLine(tree, p, f)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
lines = append(lines, l)
|
||||
}
|
||||
}
|
||||
data := filepath.Join(root.Dir, "testdata")
|
||||
if _, err := os.Stat(data); err == nil {
|
||||
err := filepath.WalkDir(data, func(path string, e fs.DirEntry, err error) error {
|
||||
if err != nil || e.IsDir() {
|
||||
return err
|
||||
}
|
||||
l, err := contentLine(tree, root, path)
|
||||
lines = append(lines, l)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
sort.Strings(lines)
|
||||
h := sha256.New()
|
||||
for _, l := range append(append([]string{}, header...), lines...) {
|
||||
h.Write([]byte(l + "\n"))
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func fileLine(tree string, p *goPackage, name string) (string, error) {
|
||||
return contentLine(tree, p, filepath.Join(p.Dir, name))
|
||||
}
|
||||
|
||||
// contentLine is "file <path> <sha256>": the path is the file's in the tree (the import path and the name when it lies
|
||||
// outside, as a local replacement does), so a digest does not depend on where the workspace is.
|
||||
func contentLine(tree string, p *goPackage, path string) (string, error) {
|
||||
label := p.ImportPath + "/" + filepath.Base(path)
|
||||
if rel, err := filepath.Rel(tree, path); err == nil && !strings.HasPrefix(rel, "..") {
|
||||
label = filepath.ToSlash(rel)
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
to, _ := os.Readlink(path)
|
||||
return "link " + label + " " + to, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return "file " + label + " " + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Issue 1291: the digests of a fixture module (testdata/repos/digest): a's tests import b and read a/testdata, c's
|
||||
// tests read an embedded file, d is reached by no test. They run the real `go list` (it loads packages, it builds
|
||||
// none), through the step runner; only the listing of the tree's go.mod files and postgres's version are answered.
|
||||
|
||||
// digestModule is a module over a copy of the fixture, in a workspace of its own.
|
||||
func digestFixture(t *testing.T) (*Module, string) {
|
||||
t.Helper()
|
||||
work := t.TempDir()
|
||||
tree := filepath.Join(work, "ws1", "tree")
|
||||
if err := os.MkdirAll(tree, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", "digest")+"/.", tree).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
m := moduleWith(newStates(), "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), work, t.TempDir()
|
||||
real := StepRunner(m.Home)
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch name {
|
||||
case "git":
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
case "postgres":
|
||||
return forgekey.Ran{Stdout: "postgres (PostgreSQL) " + strconv.Itoa(m.Recipe.PostgresMajor) + ".1\n"}
|
||||
}
|
||||
return real(ctx, env, name, args...)
|
||||
}
|
||||
return m, tree
|
||||
}
|
||||
|
||||
func digests(t *testing.T, m *Module, level string) DigestResult {
|
||||
t.Helper()
|
||||
out, err := m.digest(context.Background(), map[string]any{"workspace": "ws1", "level": level})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.(DigestResult)
|
||||
}
|
||||
|
||||
func edit(t *testing.T, tree, name, text string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(tree, name), []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1291/AC1: a file of b changes: a's digest changes (its tests import b), c's does not.
|
||||
func TestADependencysFileChangesTheDigestOfThePackageWhoseTestsImportIt(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, tree := digestFixture(t)
|
||||
before := digests(t, m, "unit")
|
||||
for _, p := range []string{"fixture/digest/a", "fixture/digest/c"} {
|
||||
if len(before.Packages[p]) != 64 {
|
||||
t.Fatalf("no digest for %s: %+v", p, before)
|
||||
}
|
||||
}
|
||||
edit(t, tree, "b/b.go", "package b\n\n// B is reached by a's tests.\nfunc B() int { return 2 + 0 }\n")
|
||||
after := digests(t, m, "unit")
|
||||
if after.Packages["fixture/digest/a"] == before.Packages["fixture/digest/a"] {
|
||||
t.Errorf("a's digest did not change when b did")
|
||||
}
|
||||
if after.Packages["fixture/digest/c"] != before.Packages["fixture/digest/c"] {
|
||||
t.Errorf("c's digest changed when b did")
|
||||
}
|
||||
if again := digests(t, m, "unit"); again.Packages["fixture/digest/a"] != after.Packages["fixture/digest/a"] {
|
||||
t.Errorf("one tree has two digests")
|
||||
}
|
||||
}
|
||||
|
||||
// 1291/AC2: a file under a's testdata/ changes: a's digest changes. An embedded file does the same for c.
|
||||
func TestATestdataFileAndAnEmbeddedFileChangeTheDigest(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, tree := digestFixture(t)
|
||||
before := digests(t, m, "unit")
|
||||
edit(t, tree, "a/testdata/x.txt", "y\n")
|
||||
after := digests(t, m, "unit")
|
||||
if after.Packages["fixture/digest/a"] == before.Packages["fixture/digest/a"] {
|
||||
t.Errorf("a's digest did not change with its testdata")
|
||||
}
|
||||
if after.Packages["fixture/digest/c"] != before.Packages["fixture/digest/c"] {
|
||||
t.Errorf("c's digest changed with a's testdata")
|
||||
}
|
||||
edit(t, tree, "c/note.txt", "another note\n")
|
||||
if again := digests(t, m, "unit"); again.Packages["fixture/digest/c"] == after.Packages["fixture/digest/c"] {
|
||||
t.Errorf("c's digest did not change with its embedded file")
|
||||
}
|
||||
}
|
||||
|
||||
// 1291/AC3: one tree at the unit and the integration level: the digests differ; at the integration level a change of the
|
||||
// recipe's postgres major (or of the nats-server version) changes it, at the unit level it does not.
|
||||
func TestTheLevelTheStoreAndTheBusAreInTheDigest(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, _ := digestFixture(t)
|
||||
unit, integration := digests(t, m, "unit"), digests(t, m, "integration")
|
||||
a := "fixture/digest/a"
|
||||
if unit.Packages[a] == integration.Packages[a] {
|
||||
t.Errorf("the unit and the integration digest of a are one")
|
||||
}
|
||||
m.Recipe.PostgresMajor++
|
||||
if got := digests(t, m, "integration"); got.Packages[a] == integration.Packages[a] {
|
||||
t.Errorf("a new postgres major left the integration digest as it was")
|
||||
}
|
||||
if got := digests(t, m, "unit"); got.Packages[a] != unit.Packages[a] {
|
||||
t.Errorf("a new postgres major moved the unit digest")
|
||||
}
|
||||
m.Recipe.PostgresMajor--
|
||||
m.Recipe.NatsServer = "9.9.9"
|
||||
if got := digests(t, m, "integration"); got.Packages[a] == integration.Packages[a] {
|
||||
t.Errorf("a new nats-server version left the integration digest as it was")
|
||||
}
|
||||
m.Recipe.NatsServer = "2.11.17"
|
||||
m.Recipe.Go = "go1.27.2"
|
||||
if got := digests(t, m, "unit"); got.Packages[a] == unit.Packages[a] {
|
||||
t.Errorf("a new Go version left the unit digest as it was")
|
||||
}
|
||||
}
|
||||
|
||||
// 1291/AC4: a change in a package no test of a reaches leaves a's digest as it was; d, with no tests, has none.
|
||||
func TestAPackageNoTestReachesChangesNoDigest(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, tree := digestFixture(t)
|
||||
before := digests(t, m, "unit")
|
||||
edit(t, tree, "d/d.go", "package d\n\n// D is reached by no test.\nfunc D() int { return 5 }\n")
|
||||
after := digests(t, m, "unit")
|
||||
for p, d := range before.Packages {
|
||||
if after.Packages[p] != d {
|
||||
t.Errorf("the digest of %s changed with a package no test of it reaches", p)
|
||||
}
|
||||
}
|
||||
if _, has := after.Packages["fixture/digest/d"]; has || strings.Join(after.WithoutTests, " ") != "fixture/digest/b fixture/digest/d" {
|
||||
t.Errorf("without tests = %v, packages = %v", after.WithoutTests, after.Packages)
|
||||
}
|
||||
}
|
||||
|
||||
// The digest does not depend on where the workspace lies.
|
||||
func TestTheDigestIsTheSameInAnotherWorkspace(t *testing.T) {
|
||||
t.Parallel()
|
||||
m1, _ := digestFixture(t)
|
||||
m2, _ := digestFixture(t)
|
||||
if d1, d2 := digests(t, m1, "unit"), digests(t, m2, "unit"); d1.Packages["fixture/digest/a"] != d2.Packages["fixture/digest/a"] {
|
||||
t.Errorf("one tree in two places has two digests")
|
||||
}
|
||||
}
|
||||
|
||||
// A module outside the tree counts by its version and checksum, not by its files: the package of mesh-sdk, as go list
|
||||
// gave it for this module's own dependency (testdata/captured/golist-external.json, which README describes).
|
||||
func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "captured", "golist-external.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkgs, err := parsePackages(string(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ext := pkgs["git.novox.be/novox/mesh-sdk/go/buildstep"]
|
||||
if ext == nil || ext.Module == nil || ext.Module.Sum == "" || ext.Module.Version == "" || ext.Module.Main {
|
||||
t.Fatalf("the capture holds no external package with a version and a checksum: %+v", ext)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
os.WriteFile(filepath.Join(dir, "r.go"), []byte("package r\n"), 0o644)
|
||||
root := &goPackage{ImportPath: "x/r", Dir: dir, Module: &goModule{Path: "x", Main: true}, GoFiles: []string{"r.go"}, TestGoFiles: []string{"r.go"}}
|
||||
testMain := &goPackage{ImportPath: "x/r.test", Deps: []string{ext.ImportPath}}
|
||||
pkgs[root.ImportPath] = root
|
||||
digest := func() string {
|
||||
d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
d1 := digest()
|
||||
ext.Module.Sum = "h1:changed="
|
||||
if d2 := digest(); d1 == d2 {
|
||||
t.Errorf("a new checksum of a module outside the tree left the digest as it was")
|
||||
}
|
||||
ext.Module.Sum, ext.Module.Version = "h1:changed=", "v9.9.9"
|
||||
if d3 := digest(); d3 == d1 {
|
||||
t.Errorf("a new version left the digest as it was")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issues 1483 and 1488 (for the indexer, issue 1444): the guard emits toolchain.reported after each write of the
|
||||
// toolchains key, with the node and the language.
|
||||
func TestTheGuardEmitsToolchainReportedAfterEachWrite(t *testing.T) {
|
||||
m := moduleWith(newStates(), "go1.27.1")
|
||||
type event struct {
|
||||
typ string
|
||||
body map[string]string
|
||||
}
|
||||
var got []event
|
||||
m.Emit = func(typ string, body any) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
var b map[string]string
|
||||
json.Unmarshal(raw, &b)
|
||||
got = append(got, event{typ, b})
|
||||
return nil
|
||||
}
|
||||
if err := m.Check(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].typ != "toolchain.reported" || got[0].body["node"] != "ace" || got[0].body["language"] != "go" || len(got[0].body) != 2 {
|
||||
t.Errorf("events = %+v, want one toolchain.reported for ace, go", got)
|
||||
}
|
||||
m.Check(context.Background())
|
||||
if len(got) != 2 {
|
||||
t.Errorf("a second guard pass emitted %d events in all, want 2", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// The event is carried by the module's stream, not named in emits (ADR 0363: one place); the validator checks that.
|
||||
func TestTheManifestDeclaresTheToolchainsStream(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Streams []struct {
|
||||
Name string `json:"name"`
|
||||
Events []string `json:"events"`
|
||||
} `json:"streams"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) != 1 || m.Streams[0].Name != "toolchains-go" || !slices.Equal(m.Streams[0].Events, []string{"toolchain.**"}) {
|
||||
t.Errorf("streams = %+v (%v)", m.Streams, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package main
|
||||
|
||||
// The fetch step (novox/hq ADR 0405 §5 §6, ADR 0466 §4 §5, ADR 0469 §2; issue 1485): a commit of a repository of
|
||||
// the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and the Go modules every
|
||||
// go.mod in it names, downloaded into the module's cache and verified against go.sum.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
|
||||
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
|
||||
const ForgeAlias = "mesh-forge"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
|
||||
// fetches and every round it checks its toolchain. A workspace lies in the module's work directory, never in its
|
||||
// home (ADR 0466 §1).
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
var (
|
||||
repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`)
|
||||
)
|
||||
|
||||
// FetchResult is what a passed fetch step says: the workspace it made, the commit in it and the directories whose
|
||||
// Go modules it downloaded.
|
||||
type FetchResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Commit string `json:"commit"`
|
||||
Modules []string `json:"modules"`
|
||||
}
|
||||
|
||||
// fetchTool is the fetch step tool.
|
||||
func (m *Module) fetchTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_go_fetch",
|
||||
Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
|
||||
"workspace of its own, and download and verify the Go modules every go.mod in it names. Answers a step id at " +
|
||||
"once; the step ends on its key in the steps state with the workspace's id.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
||||
},
|
||||
"required": []string{"repository", "commit"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.fetch,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
root := filepath.Join(m.Work, id)
|
||||
tree := filepath.Join(root, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
|
||||
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirs, err := m.goModDirs(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, d := range dirs {
|
||||
if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil {
|
||||
continue // a module that vendors resolves nothing
|
||||
}
|
||||
for _, verb := range []string{"download", "verify"} {
|
||||
r := m.Run(ctx, m.goEnv(), "go", "-C", filepath.Join(tree, d), "mod", verb)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("go mod %s in %s: %s", verb, d, firstLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr))
|
||||
}
|
||||
}
|
||||
}
|
||||
return FetchResult{Workspace: id, Commit: commit, Modules: dirs}, nil
|
||||
}
|
||||
|
||||
// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order.
|
||||
func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
|
||||
r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
dirs := []string{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
|
||||
}
|
||||
}
|
||||
sort.Strings(dirs)
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
// goEnv is what a go command of a step is given beyond the base environment: the installed toolchain and no other,
|
||||
// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by
|
||||
// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
|
||||
// that vendors, see goEnvFor.
|
||||
func (m *Module) goEnv() []string {
|
||||
return []string{
|
||||
"GOTOOLCHAIN=local",
|
||||
"GOFLAGS=-mod=mod",
|
||||
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
|
||||
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
|
||||
"GOPRIVATE=git.novox.be",
|
||||
"GIT_TERMINAL_PROMPT=0",
|
||||
"GIT_SSH_COMMAND=ssh -o BatchMode=yes",
|
||||
}
|
||||
}
|
||||
|
||||
// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor
|
||||
// directory, as fetch downloaded nothing for it, and every other by its go.mod and go.sum.
|
||||
func (m *Module) goEnvFor(dir string) []string {
|
||||
env := m.goEnv()
|
||||
if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
|
||||
env = append(env, "GOFLAGS=-mod=vendor") // the last value of a name is the one used
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// clock is the time now: Now when a test set it, else the wall clock.
|
||||
func (m *Module) clock() time.Time {
|
||||
if m.Now != nil {
|
||||
return m.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
func (m *Module) now() time.Time { return m.clock() }
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const commitSHA = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
// cmdRecord is one command a step ran.
|
||||
type cmdRecord struct {
|
||||
env []string
|
||||
line string
|
||||
}
|
||||
|
||||
// fakeStep answers the commands of a fetch: git succeeds, rev-parse answers the commit, ls-files answers the
|
||||
// go.mod files, and a go command answers by rule.
|
||||
type fakeStep struct {
|
||||
mu sync.Mutex
|
||||
calls []cmdRecord
|
||||
gomods string
|
||||
gate chan struct{} // when set, the first git command waits for it
|
||||
goRule func(line string) forgekey.Ran
|
||||
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
|
||||
}
|
||||
|
||||
func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, cmdRecord{env: env, line: line})
|
||||
f.mu.Unlock()
|
||||
if f.gate != nil && strings.Contains(line, " init ") {
|
||||
<-f.gate
|
||||
}
|
||||
switch {
|
||||
case name == "git" && strings.Contains(line, " rev-parse HEAD"):
|
||||
return forgekey.Ran{Stdout: commitSHA + "\n"}
|
||||
case name == "git" && strings.Contains(line, " ls-files "):
|
||||
return forgekey.Ran{Stdout: f.gomods}
|
||||
case name == "git" && strings.Contains(line, " checkout "):
|
||||
for i, a := range args {
|
||||
if a == "-C" {
|
||||
for _, v := range f.vendors {
|
||||
os.MkdirAll(filepath.Join(args[i+1], v, "vendor"), 0o755)
|
||||
os.WriteFile(filepath.Join(args[i+1], v, "vendor", "modules.txt"), nil, 0o644)
|
||||
}
|
||||
}
|
||||
}
|
||||
case name == "go" && f.goRule != nil:
|
||||
return f.goRule(line)
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
|
||||
func (f *fakeStep) lines() []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := []string{}
|
||||
for _, c := range f.calls {
|
||||
out = append(out, c.line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fetchModule(t *testing.T, f *fakeStep) (*Module, *keptStates) {
|
||||
t.Helper()
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Run, m.Home, m.Work, m.Cache = f.run, t.TempDir(), t.TempDir(), t.TempDir()
|
||||
return m, states
|
||||
}
|
||||
|
||||
func fetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == "build_go_fetch" {
|
||||
return tool.Run
|
||||
}
|
||||
}
|
||||
t.Fatal("no tool build_go_fetch")
|
||||
return nil
|
||||
}
|
||||
|
||||
var fetchArgs = map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}
|
||||
|
||||
// 1485/AC1: git is asked through FetchCommit with the ssh remote, and go mod download and verify run in each go.mod
|
||||
// directory (not in one that vendors) with GOTOOLCHAIN=local, GOFLAGS=-mod=mod and the module's cache paths.
|
||||
func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}}
|
||||
m, states := fetchModule(t, f)
|
||||
if _, err := fetchTool(t, m)(fetchArgs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
lines := strings.Join(f.lines(), "\n")
|
||||
for _, want := range []string{
|
||||
"remote add origin ssh://mesh-forge/novox/mesh-sdk.git",
|
||||
"fetch --no-tags --depth 1 origin " + commitSHA,
|
||||
"checkout -q --detach " + commitSHA,
|
||||
} {
|
||||
if !strings.Contains(lines, want) {
|
||||
t.Errorf("git was not asked %q:\n%s", want, lines)
|
||||
}
|
||||
}
|
||||
downloads, verifies := 0, 0
|
||||
for _, c := range f.calls {
|
||||
switch {
|
||||
case strings.HasPrefix(c.line, "go -C ") && strings.HasSuffix(c.line, " mod download"):
|
||||
downloads++
|
||||
case strings.HasPrefix(c.line, "go -C ") && strings.HasSuffix(c.line, " mod verify"):
|
||||
verifies++
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.line, "vend") {
|
||||
t.Errorf("a module that vendors was downloaded: %s", c.line)
|
||||
}
|
||||
env := strings.Join(c.env, " ")
|
||||
for _, want := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be"} {
|
||||
if !strings.Contains(env, want) {
|
||||
t.Errorf("%s: environment lacks %s: %s", c.line, want, env)
|
||||
}
|
||||
}
|
||||
}
|
||||
if downloads != 2 || verifies != 2 {
|
||||
t.Errorf("%d downloads and %d verifies, want 2 each (go.mod and sub/dir/go.mod):\n%s", downloads, verifies, lines)
|
||||
}
|
||||
var res FetchResult
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Commit != commitSHA || strings.Join(res.Modules, ",") != ".,sub/dir,vend" || res.Workspace == "" {
|
||||
t.Errorf("result = %s", st.Result)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, res.Workspace, "tree")); err != nil {
|
||||
t.Errorf("the workspace is not in the work directory: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1485/AC2: a variable such as MESH_REGISTRY_PASSWORD or a *_TOKEN of the process is seen by no command of the step:
|
||||
// the runner builds the environment from scratch, and the step passes it nothing of the kind.
|
||||
func TestNoCommandOfTheStepSeesAPasswordOrAToken(t *testing.T) {
|
||||
t.Setenv("MESH_REGISTRY_PASSWORD", "hunter2-registry")
|
||||
t.Setenv("FORGE_TOKEN", "hunter2-token")
|
||||
t.Setenv("HOME", "/not/the/account")
|
||||
r := StepRunner("/home/account")
|
||||
got := r(context.Background(), []string{"EXTRA=1"}, "env")
|
||||
if got.Status != 0 {
|
||||
t.Fatalf("env: %+v", got)
|
||||
}
|
||||
for _, secret := range []string{"hunter2", "MESH_REGISTRY_PASSWORD", "FORGE_TOKEN", "/not/the/account"} {
|
||||
if strings.Contains(got.Stdout, secret) {
|
||||
t.Errorf("a command saw %q:\n%s", secret, got.Stdout)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"HOME=/home/account", "EXTRA=1", "PATH="} {
|
||||
if !strings.Contains(got.Stdout, want) {
|
||||
t.Errorf("a command lacks %q:\n%s", want, got.Stdout)
|
||||
}
|
||||
}
|
||||
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
m, states := fetchModule(t, f)
|
||||
fetchTool(t, m)(fetchArgs)
|
||||
states.waitEnd(t)
|
||||
for _, c := range f.calls {
|
||||
for _, w := range c.env {
|
||||
if k, _, _ := strings.Cut(w, "="); strings.HasSuffix(k, "TOKEN") || strings.HasSuffix(k, "PASSWORD") || strings.HasSuffix(k, "SECRET") {
|
||||
t.Errorf("%s was given %s", c.line, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1485/AC3: go mod verify failing ends the step failed naming the directory and the first line of its output.
|
||||
func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", goRule: func(line string) forgekey.Ran {
|
||||
if strings.HasSuffix(line, "tools mod verify") || strings.Contains(line, "/tools mod verify") {
|
||||
return forgekey.Ran{Status: 1, Stdout: "\nexample.org/x v1.0.0: dir has been modified\nsecond line\n"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}}
|
||||
m, states := fetchModule(t, f)
|
||||
if _, err := fetchTool(t, m)(fetchArgs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepFailed {
|
||||
t.Fatalf("the step ended %s, want failed", st.State)
|
||||
}
|
||||
if !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, "example.org/x v1.0.0: dir has been modified") || strings.Contains(st.Reason, "second line") {
|
||||
t.Errorf("reason = %q", st.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// 1485/AC4: the call answers a step id before the fetch ends, and the key then ends passed with the workspace.
|
||||
func TestTheCallAnswersAStepIdBeforeTheFetchEnds(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n", gate: make(chan struct{})}
|
||||
m, states := fetchModule(t, f)
|
||||
out, err := fetchTool(t, m)(fetchArgs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answer := out.(map[string]any)
|
||||
id, _ := answer["step"].(string)
|
||||
if id == "" || answer["key"] != "ace."+id {
|
||||
t.Fatalf("answer = %v, want a step id and its key", answer)
|
||||
}
|
||||
select {
|
||||
case <-states.ended:
|
||||
t.Fatal("the step ended while git was held")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
states.mu.Lock()
|
||||
running := states.steps["ace."+id]
|
||||
states.mu.Unlock()
|
||||
if len(running) != 1 || running[0].State != delivery.StepRunning {
|
||||
t.Errorf("the key holds %+v, want one running step", running)
|
||||
}
|
||||
close(f.gate)
|
||||
select {
|
||||
case key := <-states.ended:
|
||||
if key != "ace."+id {
|
||||
t.Errorf("ended %s", key)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the step never ended")
|
||||
}
|
||||
states.mu.Lock()
|
||||
last := states.steps["ace."+id][len(states.steps["ace."+id])-1]
|
||||
states.mu.Unlock()
|
||||
var res FetchResult
|
||||
if last.State != delivery.StepPassed || json.Unmarshal(last.Result, &res) != nil || res.Workspace == "" {
|
||||
t.Errorf("the key ends %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepositoryThatIsNotOwnerSlashNameIsRefusedBeforeAnythingRuns(t *testing.T) {
|
||||
for _, repo := range []string{"", "mesh-sdk", "../etc/passwd", "a/b/c", "a/..", "-o/x", "https://x/y"} {
|
||||
f := &fakeStep{}
|
||||
m, states := fetchModule(t, f)
|
||||
fetchTool(t, m)(map[string]any{"repository": repo, "commit": commitSHA})
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepFailed || len(f.lines()) != 0 {
|
||||
t.Errorf("%q: the step ended %s after %d commands", repo, st.State, len(f.lines()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Workspaces older than a day are removed; younger ones, and anything that is not a workspace, stay.
|
||||
func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) {
|
||||
m := &Module{Work: t.TempDir()}
|
||||
for name, age := range map[string]time.Duration{"old1": 25 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} {
|
||||
dir := filepath.Join(m.Work, name)
|
||||
os.MkdirAll(filepath.Join(dir, "tree"), 0o755)
|
||||
when := time.Now().Add(-age)
|
||||
os.Chtimes(dir, when, when)
|
||||
}
|
||||
m.CleanWorkspaces(time.Now())
|
||||
for name, want := range map[string]bool{"old1": false, "young": true, "Keep-This": true} {
|
||||
_, err := os.Stat(filepath.Join(m.Work, name))
|
||||
if (err == nil) != want {
|
||||
t.Errorf("%s present = %v, want %v", name, err == nil, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The work and cache directories the code uses are the ones the manifest places, and neither is in the home.
|
||||
func TestWorkAndCacheAreTheDirectoriesTheManifestPlacesOutsideTheHome(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
home := resource(t, m, "account")["home"].(string)
|
||||
for id, path := range map[string]string{"work": WorkDir, "cache": CacheDir} {
|
||||
if resource(t, m, id)["path"] != path {
|
||||
t.Errorf("%s: the manifest places %v, the code uses %s", id, resource(t, m, id)["path"], path)
|
||||
}
|
||||
if path == home || strings.HasPrefix(path, home+"/") {
|
||||
t.Errorf("%s %s is in the home %s", id, path, home)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
package main
|
||||
|
||||
// The forge side of a build module (novox/hq ADR 0466 §2 and §4, issue 1484). This file is the same for every build
|
||||
// module: mesh-build-node takes a copy and changes the three constants below.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const (
|
||||
forgeModule = "mesh-build-go"
|
||||
forgeHome = "/var/lib/mesh-build-go"
|
||||
forgeModuleD = "../.." // the module's directory, where module.json and files/ are
|
||||
)
|
||||
|
||||
type forgeManifest struct {
|
||||
Requires []any `json:"requires"`
|
||||
State []any `json:"state"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
func readForgeManifest(t *testing.T) forgeManifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(forgeModuleD, "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m forgeManifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func forgeTemplate(t *testing.T, r map[string]any) string {
|
||||
t.Helper()
|
||||
name, _ := r["template"].(string)
|
||||
if name == "" {
|
||||
t.Fatalf("resource %v names no template", r["id"])
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(forgeModuleD, filepath.FromSlash(name)))
|
||||
if err != nil {
|
||||
t.Fatalf("resource %v names %s: %v", r["id"], name, err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func forgeResource(t *testing.T, m forgeManifest, id string) map[string]any {
|
||||
t.Helper()
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
if r["owner"] != forgeModule {
|
||||
t.Errorf("%s is owned by %v, not the module's own account", id, r["owner"])
|
||||
}
|
||||
if p, _ := r["path"].(string); !strings.HasPrefix(p, "${dir:") || strings.Contains(p, "account-home") {
|
||||
t.Errorf("%s is at %q: a path of the account's own home (${dir:...}), never the operator's", id, p)
|
||||
}
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("no resource %q", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
var forgeSample = map[string]string{
|
||||
"known_host": "[novox.internal]:222 ssh-ed25519 AAAAC3Nza",
|
||||
"host_key": "ssh-ed25519 AAAAC3Nza",
|
||||
"ssh_host": "novox.internal",
|
||||
"ssh_alias": "mesh-forge",
|
||||
"ssh_port": "222",
|
||||
"ssh_base": "ssh://git@novox.internal:222/",
|
||||
"https_base": "https://git.novox.be/",
|
||||
}
|
||||
|
||||
// forgeRender fills ${bound:forge-account:<key>} with the sample values, as the provisioner client does.
|
||||
func forgeRender(t *testing.T, s string) string {
|
||||
t.Helper()
|
||||
for k, v := range forgeSample {
|
||||
s = strings.ReplaceAll(s, "${bound:forge-account:"+k+"}", v)
|
||||
}
|
||||
if strings.Contains(s, "${") {
|
||||
t.Errorf("unfilled placeholder in %q", s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// 1484/AC1: given sample bound values, the ssh config has Host mesh-forge, IdentitiesOnly yes and the key under the
|
||||
// module's own home; known_hosts carries the host key; git's config rewrites https_base to ssh://mesh-forge/; no
|
||||
// ${ is left.
|
||||
func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
|
||||
cfg := forgeResource(t, m, "ssh-config")
|
||||
want := "Host mesh-forge\n HostName novox.internal\n Port 222\n HostKeyAlias mesh-forge\n User git\n IdentityFile ~/.ssh/mesh_ed25519\n IdentitiesOnly yes\n"
|
||||
if got := forgeRender(t, forgeTemplate(t, cfg)); got != want {
|
||||
t.Errorf("ssh config = %q, want %q", got, want)
|
||||
}
|
||||
if cfg["path"] != "${dir:ssh-dir}/config" || cfg["mode"] != "0600" || cfg["into"] != nil {
|
||||
t.Errorf("ssh config resource = %v, want the whole file ${dir:ssh-dir}/config at mode 0600", cfg)
|
||||
}
|
||||
|
||||
kh := forgeResource(t, m, "known-hosts")
|
||||
if got := strings.TrimSpace(forgeRender(t, forgeTemplate(t, kh))); got != "mesh-forge ssh-ed25519 AAAAC3Nza" {
|
||||
t.Errorf("known_hosts = %q", got)
|
||||
}
|
||||
|
||||
g := forgeResource(t, m, "git-config")
|
||||
wantGit := "[url \"ssh://mesh-forge/\"]\n\tinsteadOf = https://git.novox.be/\n"
|
||||
if got := forgeRender(t, forgeTemplate(t, g)); got != wantGit {
|
||||
t.Errorf("git config = %q, want %q", got, wantGit)
|
||||
}
|
||||
|
||||
dir := forgeResource(t, m, "ssh-dir")
|
||||
if dir["mode"] != "0700" || dir["path"] != "${dir:state}/.ssh" {
|
||||
t.Errorf("ssh-dir = %v, want ${dir:state}/.ssh at mode 0700", dir)
|
||||
}
|
||||
}
|
||||
|
||||
// 1484/AC2: the module requires forge-account, keeps the state keys, and every file resource takes its text from a
|
||||
// template in the files/ archive (novox/hq ADR 0349 §2).
|
||||
func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
if len(m.Requires) != 1 || m.Requires[0] != "forge-account" {
|
||||
t.Errorf("requires = %v, want [forge-account]", m.Requires)
|
||||
}
|
||||
keeps := false
|
||||
for _, s := range m.State {
|
||||
if s == "keys" {
|
||||
keeps = true
|
||||
}
|
||||
}
|
||||
if !keeps {
|
||||
t.Errorf("state = %v lacks keys", m.State)
|
||||
}
|
||||
files := 0
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline {
|
||||
t.Errorf("resource %v keeps its text inline", r["id"])
|
||||
}
|
||||
if r["type"] == "file" {
|
||||
files++
|
||||
forgeTemplate(t, r)
|
||||
}
|
||||
}
|
||||
if files == 0 {
|
||||
t.Errorf("no file resource: the forge's ssh files are missing")
|
||||
}
|
||||
packs := 0
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a["kind"] == "archive" && a["from"] == "files" && a["name"] == "files" {
|
||||
packs++
|
||||
}
|
||||
}
|
||||
if packs != 1 {
|
||||
t.Errorf("the build packs files/ into %d archives named files, not one", packs)
|
||||
}
|
||||
}
|
||||
|
||||
// forgeEd25519Line is a public key line with a well-formed ed25519 blob.
|
||||
func forgeEd25519Line(comment string) string {
|
||||
blob := []byte{}
|
||||
for _, part := range [][]byte{[]byte("ssh-ed25519"), make([]byte, 32)} {
|
||||
l := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(l, uint32(len(part)))
|
||||
blob = append(append(blob, l...), part...)
|
||||
}
|
||||
return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(blob) + " " + comment
|
||||
}
|
||||
|
||||
type forgeKeys struct{ puts map[string][]byte }
|
||||
|
||||
func (f *forgeKeys) Put(key string, value any) (uint64, error) {
|
||||
raw, _ := json.Marshal(value)
|
||||
if f.puts == nil {
|
||||
f.puts = map[string][]byte{}
|
||||
}
|
||||
f.puts[key] = raw
|
||||
return uint64(len(f.puts)), nil
|
||||
}
|
||||
|
||||
// forgeKeygen stands in for ssh-keygen: it writes the pair the command names.
|
||||
func forgeKeygen(_ context.Context, _ []string, name string, args ...string) forgekey.Ran {
|
||||
var file, comment string
|
||||
for i, a := range args {
|
||||
switch a {
|
||||
case "-f":
|
||||
file = args[i+1]
|
||||
case "-C":
|
||||
comment = args[i+1]
|
||||
}
|
||||
}
|
||||
if name != "ssh-keygen" || file == "" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "unexpected"}
|
||||
}
|
||||
os.WriteFile(file, []byte("PRIVATE\n"), 0o600)
|
||||
os.WriteFile(file+".pub", []byte(forgeEd25519Line(comment)+"\n"), 0o644)
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
|
||||
// 1484/AC3: given the bundle started with an empty home, the keeper writes <node>.<module> with kind builder, and
|
||||
// the key is in the home's .ssh.
|
||||
func TestTheKeeperWritesTheModulesEntryWithKindBuilder(t *testing.T) {
|
||||
home, st := t.TempDir(), &forgeKeys{}
|
||||
if err := keeperFor(home, "ace", forgeModule, forgeKeygen, st).Once(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(home, ".ssh", "mesh_ed25519")); err != nil {
|
||||
t.Errorf("no key in an empty home: %v", err)
|
||||
}
|
||||
raw, ok := st.puts["ace."+forgeModule]
|
||||
if !ok || len(st.puts) != 1 {
|
||||
t.Fatalf("entries = %v, want one at ace.%s", st.puts, forgeModule)
|
||||
}
|
||||
var e forgekey.KeyEntry
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e.Kind != "builder" || e.Node != "ace" || e.Account != forgeModule || !strings.HasPrefix(e.Public, "ssh-ed25519 ") {
|
||||
t.Errorf("entry = %s", raw)
|
||||
}
|
||||
if strings.Contains(strings.ToUpper(string(raw)), "PRIVATE") {
|
||||
t.Errorf("the entry carries private material: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// The home the manifest places is the one the account is given, so the keeper's home and the templates' agree.
|
||||
func TestTheAccountsHomeIsTheOneTheManifestNames(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "user" && r["name"] == forgeModule && r["home"] != forgeHome {
|
||||
t.Errorf("the account's home is %v, want %s", r["home"], forgeHome)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
// The integration level (novox/hq ADR 0445 §2 §3, ADR 0470; issue 1486): `go test -tags integration` over the
|
||||
// workspace, against a throwaway postgres and a throwaway bus started for this run alone, handed to the tests as
|
||||
// MESH_TEST_POSTGRES and MESH_TEST_NATS. Its setup (compiling, the store, the bus) and its running time are measured and
|
||||
// judged apart, 30 s and 60 s; both servers are stopped and the run's directory removed whatever the tests did.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// The integration level's bounds (ADR 0445 §3).
|
||||
const (
|
||||
IntegrationSetup = 30 * time.Second
|
||||
IntegrationRun = 60 * time.Second
|
||||
)
|
||||
|
||||
func (m *Module) integrationBounds() Bounds {
|
||||
b := m.IntegrationBounds
|
||||
if b.Setup <= 0 {
|
||||
b.Setup = IntegrationSetup
|
||||
}
|
||||
if b.Run <= 0 {
|
||||
b.Run = IntegrationRun
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string) (res *TestResult, err error) {
|
||||
major, err := m.checkPostgres(ctx)
|
||||
if err != nil {
|
||||
return nil, err // a *buildstep.Refusal: this level only
|
||||
}
|
||||
run := filepath.Join(m.Work, workspace, "run-"+buildstep.NewID())
|
||||
if err := os.MkdirAll(run, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The run's directory goes, and the servers stop, however the level ends: passed, failed, a panic.
|
||||
defer os.RemoveAll(run)
|
||||
|
||||
began := m.clock()
|
||||
dirs, err := m.goModDirs(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
postgres, stopStore, err := m.startStore(ctx, run)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the integration level's store: %w", err)
|
||||
}
|
||||
defer stopStore()
|
||||
nats, stopBus, err := m.startBus(ctx, filepath.Join(run, "nats"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the integration level's bus: %w", err)
|
||||
}
|
||||
defer stopBus()
|
||||
env := []string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}
|
||||
// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
|
||||
// below finds them built. Whether they compile is for the run to say; its failure is not read here.
|
||||
for _, d := range dirs {
|
||||
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", "integration", "-run", "^$", "-count=1", "./...")
|
||||
}
|
||||
setup := m.clock().Sub(began)
|
||||
|
||||
return m.runLevel(ctx, tree, levelOpts{
|
||||
level: LevelIntegration, tags: []string{"integration"}, packages: packages, bounds: m.integrationBounds(),
|
||||
env: env,
|
||||
setup: setup, wall: true,
|
||||
store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Issue 1486: the integration level, with the store and the bus faked at the process boundary: the commands the level
|
||||
// runs are recorded, and a clock the commands advance stands in for the time they would take.
|
||||
|
||||
type integFake struct {
|
||||
mu sync.Mutex
|
||||
calls []cmdRecord
|
||||
clock time.Time
|
||||
postgres string // what `postgres --version` prints
|
||||
captured string // the go test -json output of the run
|
||||
// advance is how long a phase takes on the fake clock: compile, start (of the store), test.
|
||||
advance map[string]time.Duration
|
||||
busURL string
|
||||
busDir string
|
||||
busDown bool
|
||||
runDirs []string // run directories seen when the store was started, to be gone at the end
|
||||
panicOn string // a phase that panics
|
||||
}
|
||||
|
||||
func newIntegFake(t *testing.T, captured string) *integFake {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "captured", captured+".json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &integFake{clock: time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC), postgres: "postgres (PostgreSQL) 18.6",
|
||||
captured: string(raw), advance: map[string]time.Duration{}, busURL: "nats://127.0.0.1:45222"}
|
||||
}
|
||||
|
||||
func (f *integFake) now() time.Time { f.mu.Lock(); defer f.mu.Unlock(); return f.clock }
|
||||
|
||||
func (f *integFake) tick(phase string) {
|
||||
f.mu.Lock()
|
||||
f.clock = f.clock.Add(f.advance[phase])
|
||||
f.mu.Unlock()
|
||||
if f.panicOn == phase {
|
||||
panic("the " + phase + " phase panicked")
|
||||
}
|
||||
}
|
||||
|
||||
func (f *integFake) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, cmdRecord{env: env, line: line})
|
||||
f.mu.Unlock()
|
||||
switch {
|
||||
case name == "postgres":
|
||||
return forgekey.Ran{Stdout: f.postgres + "\n"}
|
||||
case name == "git" && len(args) > 2 && args[2] == "ls-files":
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
case name == "go" && strings.Contains(line, "-run ^$"):
|
||||
f.tick("compile")
|
||||
case name == "go":
|
||||
f.tick("test")
|
||||
status := 0
|
||||
if strings.Contains(f.captured, `"Action":"fail"`) {
|
||||
status = 1
|
||||
}
|
||||
return forgekey.Ran{Stdout: f.captured, Status: status}
|
||||
case name == "pg_ctl" && strings.HasSuffix(line, " start"):
|
||||
f.tick("start")
|
||||
for i, a := range args {
|
||||
if a == "-D" {
|
||||
f.runDirs = append(f.runDirs, filepath.Dir(filepath.Dir(args[i+1])))
|
||||
}
|
||||
}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
|
||||
func (f *integFake) startBus(_ context.Context, dir string) (string, func(), error) {
|
||||
f.mu.Lock()
|
||||
f.busDir = dir
|
||||
f.mu.Unlock()
|
||||
return f.busURL, func() { f.mu.Lock(); f.busDown = true; f.mu.Unlock() }, nil
|
||||
}
|
||||
|
||||
func (f *integFake) lines() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := []string{}
|
||||
for _, c := range f.calls {
|
||||
out = append(out, c.line)
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
func integModule(t *testing.T, f *integFake) (*Module, *keptStates) {
|
||||
t.Helper()
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
|
||||
m.Run, m.Now, m.StartBus = f.run, f.now, f.startBus
|
||||
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, states
|
||||
}
|
||||
|
||||
func runLevelTool(t *testing.T, m *Module, states *keptStates, level string) (TestResult, delivery.Step) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name != "build_go_test" {
|
||||
continue
|
||||
}
|
||||
if _, err := tool.Run(map[string]any{"workspace": "ws1", "level": level}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res TestResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
// 1486/AC2: the store's start carries listen_addresses=” and a socket directory under the run; the tests are given
|
||||
// its address and the bus's, and the run's directory is in the workspace's.
|
||||
func TestTheStoreListensOnASocketUnderTheRunAndNowhereElse(t *testing.T) {
|
||||
f := newIntegFake(t, "slow")
|
||||
m, states := integModule(t, f)
|
||||
res, st := runLevelTool(t, m, states, "integration")
|
||||
if st.State != delivery.StepPassed || !res.Passed || res.Level != "integration" {
|
||||
t.Fatalf("the step ended %s (%s): %+v", st.State, st.Reason, res)
|
||||
}
|
||||
var start, initdb string
|
||||
for _, c := range f.calls {
|
||||
if strings.HasPrefix(c.line, "pg_ctl ") && strings.HasSuffix(c.line, " start") {
|
||||
start = c.line
|
||||
}
|
||||
if strings.HasPrefix(c.line, "initdb ") {
|
||||
initdb = c.line
|
||||
}
|
||||
}
|
||||
run := f.runDirs[0]
|
||||
if !strings.HasPrefix(run, filepath.Join(m.Work, "ws1", "run-")) {
|
||||
t.Errorf("the run directory is %s, not under the workspace's", run)
|
||||
}
|
||||
_, afterK, _ := strings.Cut(start, "-k ")
|
||||
socket, _, _ := strings.Cut(afterK, " ")
|
||||
if len(socket)+len(socketName) >= 100 {
|
||||
t.Errorf("the socket %q is too long for postgres", socket)
|
||||
}
|
||||
for _, want := range []string{"listen_addresses=''", "-k " + socket, "fsync=off", "-w"} {
|
||||
if !strings.Contains(start, want) {
|
||||
t.Errorf("the store's start %q lacks %q", start, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(start, "listen_addresses='*'") || strings.Contains(start, "-h ") || strings.Contains(start, "port=") {
|
||||
t.Errorf("the store listens on TCP: %s", start)
|
||||
}
|
||||
if !strings.Contains(initdb, "-D "+filepath.Join(run, "pg", "data")) || !strings.Contains(initdb, "--auth=trust") {
|
||||
t.Errorf("initdb = %q", initdb)
|
||||
}
|
||||
var test cmdRecord
|
||||
for _, c := range f.calls {
|
||||
if strings.HasPrefix(c.line, "go ") && strings.Contains(c.line, "-json") {
|
||||
test = c
|
||||
}
|
||||
}
|
||||
if !strings.Contains(test.line, "-tags integration") {
|
||||
t.Errorf("the run does not ask for the tag: %s", test.line)
|
||||
}
|
||||
env := map[string]string{}
|
||||
for _, w := range test.env {
|
||||
k, v, _ := strings.Cut(w, "=")
|
||||
env[k] = v
|
||||
}
|
||||
u, err := url.Parse(env["MESH_TEST_POSTGRES"])
|
||||
if err != nil || u.Scheme != "postgres" || u.Query().Get("host") != socket || u.Host != "" {
|
||||
t.Errorf("MESH_TEST_POSTGRES = %q", env["MESH_TEST_POSTGRES"])
|
||||
}
|
||||
if strings.Count(strings.TrimPrefix(env["MESH_TEST_POSTGRES"], "postgres://"), "/") != 1 {
|
||||
t.Errorf("MESH_TEST_POSTGRES = %q has a slash beyond the one before the database: the controller's tests cut it at the last", env["MESH_TEST_POSTGRES"])
|
||||
}
|
||||
if env["MESH_TEST_NATS"] != "nats://127.0.0.1:45222" || f.busDir != filepath.Join(run, "nats") {
|
||||
t.Errorf("MESH_TEST_NATS = %q, bus dir %q", env["MESH_TEST_NATS"], f.busDir)
|
||||
}
|
||||
if res.Store != "postgres 18" || res.Bus != "nats-server "+m.Recipe.NatsServer {
|
||||
t.Errorf("the result names %q and %q", res.Store, res.Bus)
|
||||
}
|
||||
}
|
||||
|
||||
// 1486/AC3: a postgres of major 17 refuses the integration level, naming 17 and 18; the unit level still runs.
|
||||
func TestAnotherPostgresMajorRefusesTheIntegrationLevelOnly(t *testing.T) {
|
||||
f := newIntegFake(t, "slow")
|
||||
f.postgres = "postgres (PostgreSQL) 17.2"
|
||||
m, states := integModule(t, f)
|
||||
_, st := runLevelTool(t, m, states, "integration")
|
||||
if st.State != delivery.StepRefused || !strings.Contains(st.Reason, "17") || !strings.Contains(st.Reason, "18") {
|
||||
t.Fatalf("the integration step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
if strings.Contains(f.lines(), "initdb") || strings.Contains(f.lines(), "-tags") {
|
||||
t.Errorf("the refused level ran something:\n%s", f.lines())
|
||||
}
|
||||
res, st := runLevelTool(t, m, states, "unit")
|
||||
if st.State != delivery.StepPassed || !res.Passed || res.Level != "unit" {
|
||||
t.Errorf("the unit level: %s %+v", st.State, res)
|
||||
}
|
||||
}
|
||||
|
||||
// 1486/AC4: a setup over thirty seconds fails the level naming the setup bound; a run over sixty names the running
|
||||
// bound and the slowest tests. The clock the commands move stands in for the time they take.
|
||||
func TestSetupAndRunningTimeAreJudgedApart(t *testing.T) {
|
||||
f := newIntegFake(t, "slow")
|
||||
f.advance["compile"], f.advance["start"] = 20*time.Second, 15*time.Second
|
||||
m, states := integModule(t, f)
|
||||
res, _ := runLevelTool(t, m, states, "integration")
|
||||
if !res.Passed || !res.OverBound || res.OverWhich != "setup" || res.Setup != 35 || !strings.Contains(res.Why, "setup took 35.00 s, over its setup bound of 30 s") || strings.Contains(res.Why, "slowest tests") {
|
||||
t.Errorf("a setup of 35 s: %+v", res)
|
||||
}
|
||||
|
||||
g := newIntegFake(t, "slow")
|
||||
g.advance["test"] = 61 * time.Second
|
||||
m, states = integModule(t, g)
|
||||
res, _ = runLevelTool(t, m, states, "integration")
|
||||
if !res.Passed || res.OverWhich != "run" || res.Elapsed != 61 || !strings.Contains(res.Why, "took 61.00 s, over its bound of 60 s; the slowest tests: fixture/slow TestTakesAWhile") || strings.Contains(res.Why, "setup") {
|
||||
t.Errorf("a run of 61 s: %+v", res)
|
||||
}
|
||||
|
||||
h := newIntegFake(t, "slow")
|
||||
h.advance["compile"], h.advance["test"] = 29*time.Second, 59*time.Second
|
||||
m, states = integModule(t, h)
|
||||
if res, _ = runLevelTool(t, m, states, "integration"); !res.Passed || res.OverBound || res.Setup != 29 || res.Elapsed != 59 {
|
||||
t.Errorf("29 s of setup and 59 s of running are within both bounds: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// 1486/AC5: a run that fails, and a run that panics: the store is stopped, the bus is stopped, the run's directory is
|
||||
// gone.
|
||||
func TestAFailedOrPanickedRunStopsTheServersAndRemovesTheRunDirectory(t *testing.T) {
|
||||
f := newIntegFake(t, "failing")
|
||||
m, states := integModule(t, f)
|
||||
res, _ := runLevelTool(t, m, states, "integration")
|
||||
if res.Passed {
|
||||
t.Fatalf("the failing fixture passed: %+v", res)
|
||||
}
|
||||
checkCleaned := func(f *integFake, m *Module) {
|
||||
t.Helper()
|
||||
lines := f.lines()
|
||||
if !strings.Contains(lines, "pg_ctl -D "+filepath.Join(f.runDirs[0], "pg", "data")+" -m immediate -w stop") {
|
||||
t.Errorf("the store was not stopped:\n%s", lines)
|
||||
}
|
||||
if strings.LastIndex(lines, " stop") < strings.LastIndex(lines, "-json") {
|
||||
t.Errorf("the store was stopped before the tests ended:\n%s", lines)
|
||||
}
|
||||
if !f.busDown {
|
||||
t.Errorf("the bus was not stopped")
|
||||
}
|
||||
if _, err := os.Stat(f.runDirs[0]); !os.IsNotExist(err) {
|
||||
t.Errorf("the run directory %s is still there (%v)", f.runDirs[0], err)
|
||||
}
|
||||
}
|
||||
checkCleaned(f, m)
|
||||
|
||||
g := newIntegFake(t, "failing")
|
||||
g.panicOn = "test"
|
||||
m, _ = integModule(t, g)
|
||||
func() {
|
||||
defer func() { recover() }()
|
||||
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil)
|
||||
}()
|
||||
checkCleaned(g, m)
|
||||
}
|
||||
|
||||
// A store that does not start ends the step failed, with the server stopped and the directory gone.
|
||||
func TestAStoreThatDoesNotStartEndsTheStepFailedAndLeavesNothing(t *testing.T) {
|
||||
f := newIntegFake(t, "slow")
|
||||
m, states := integModule(t, f)
|
||||
inner := m.Run
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
r := inner(ctx, env, name, args...)
|
||||
if name == "pg_ctl" && strings.HasSuffix(strings.Join(args, " "), " start") {
|
||||
return forgekey.Ran{Status: 1, Stderr: "pg_ctl: could not start server\n"}
|
||||
}
|
||||
return r
|
||||
}
|
||||
_, st := runLevelTool(t, m, states, "integration")
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "pg_ctl: could not start server") {
|
||||
t.Errorf("the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
if !strings.Contains(f.lines(), " stop") {
|
||||
t.Errorf("the half-started store was not stopped:\n%s", f.lines())
|
||||
}
|
||||
if f.busDir != "" {
|
||||
t.Errorf("the bus was started after the store failed")
|
||||
}
|
||||
if entries, _ := os.ReadDir(filepath.Join(m.Work, "ws1")); len(entries) != 1 || entries[0].Name() != "tree" {
|
||||
t.Errorf("the workspace holds %v, want only its tree", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThePostgresMajorIsReadFromTheVersionLine(t *testing.T) {
|
||||
for line, want := range map[string]int{"postgres (PostgreSQL) 18.6": 18, "postgres (PostgreSQL) 9.6.24": 9, "postgres (PostgreSQL) 17.2 (Arch)": 17} {
|
||||
f := &integFake{postgres: line, advance: map[string]time.Duration{}}
|
||||
m := &Module{Run: f.run}
|
||||
if got, err := m.postgresMajor(context.Background()); err != nil || got != want {
|
||||
t.Errorf("%q = %d, %v; want %d", line, got, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := (&Module{Run: (&integFake{postgres: "no", advance: map[string]time.Duration{}}).run}).postgresMajor(context.Background()); err == nil {
|
||||
t.Errorf("accepted a line that is not postgres's")
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1486 review: postgres refuses a socket path of 108 bytes and over. A run directory deep in the work directory
|
||||
// gets a short directory of its own for the socket, removed with the store; a short one keeps it under the run.
|
||||
func TestTheSocketPathStaysUnderTheLimitForALongRunDirectory(t *testing.T) {
|
||||
long := filepath.Join(t.TempDir(), strings.Repeat("deep", 20), "ws1", "run-mvh5uc96142806")
|
||||
if err := os.MkdirAll(long, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir, short, err := socketDir(long)
|
||||
if err != nil || !short || len(dir)+len(socketName) >= 100 {
|
||||
t.Fatalf("socketDir(long) = %q, %v, %v", dir, short, err)
|
||||
}
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
t.Errorf("the short directory was not made: %v", err)
|
||||
}
|
||||
os.RemoveAll(dir)
|
||||
|
||||
near := t.TempDir()
|
||||
if dir, short, err := socketDir(near); err != nil || (len(near) < 50 && (short || !strings.HasPrefix(dir, near))) {
|
||||
t.Errorf("socketDir(short) = %q, %v, %v", dir, short, err)
|
||||
}
|
||||
|
||||
// Through the store: the short directory is removed when the store is stopped.
|
||||
f := newIntegFake(t, "slow")
|
||||
m, _ := integModule(t, f)
|
||||
_, stop, err := m.startStore(context.Background(), long)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sock string
|
||||
for _, c := range f.calls {
|
||||
if _, a, ok := strings.Cut(c.line, "-k "); ok && strings.HasPrefix(c.line, "pg_ctl") {
|
||||
sock, _, _ = strings.Cut(a, " ")
|
||||
}
|
||||
}
|
||||
if sock == "" || len(sock)+len(socketName) >= 100 {
|
||||
t.Fatalf("the store was started with the socket %q", sock)
|
||||
}
|
||||
stop()
|
||||
if _, err := os.Stat(sock); !os.IsNotExist(err) {
|
||||
t.Errorf("the short socket directory %s outlived the store (%v)", sock, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
// The module's own forge key (novox/hq ADR 0466 §2, issue 1484): mesh-sdk's key keeper, kind builder, for the
|
||||
// module's account in its home. It makes ~/.ssh/mesh_ed25519 when it is missing and writes one entry to this
|
||||
// module's `keys` state at <node>.<account>, which gitea's provider reads to register the key on the forge user
|
||||
// that reads and pushes nothing (ADR 0466 §3).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// keeperFor is this module's keeper: go/forgekey's, of kind builder, for account in home on node, whose entry lies
|
||||
// at <node>.<account> of the keys state.
|
||||
func keeperFor(home, node, account string, run forgekey.Runner, state forgekey.KeyPutter) *forgekey.KeyKeeper {
|
||||
return &forgekey.KeyKeeper{Home: home, Node: node, Account: account, Kind: forgekey.KindBuilder, Run: run, State: state, Now: time.Now}
|
||||
}
|
||||
|
||||
// startKeeper runs the keeper when the runtime launched this process for a node: outside a launch — a build, a
|
||||
// check — there is no state to put the entry in.
|
||||
func startKeeper(ctx context.Context) {
|
||||
node := strings.TrimSpace(os.Getenv("MESH_NODE"))
|
||||
u, err := user.Current()
|
||||
if node == "" || err != nil || u.HomeDir == "" {
|
||||
return
|
||||
}
|
||||
k := keeperFor(u.HomeDir, node, u.Username, forgekey.ExecRunner, keysState{})
|
||||
k.Keep(ctx, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-go] "+f+"\n", a...) })
|
||||
}
|
||||
|
||||
type keysState struct{}
|
||||
|
||||
func (keysState) Put(key string, value any) (uint64, error) {
|
||||
return stdio.State("keys").Put(key, value)
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package main
|
||||
|
||||
// The judging of a test level (novox/hq ADR 0445 §2 §3; issues 1290 and 1486): `go test -json` read into a result
|
||||
// that says, per package, what ran and for how long, which tests failed and why, and whether the level kept its
|
||||
// bound. A level that failed is still a finished step: the verdict is in the result, and a step that ends failed is a
|
||||
// step that could not run.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The two levels (ADR 0445 §2).
|
||||
const (
|
||||
LevelUnit = "unit"
|
||||
LevelIntegration = "integration"
|
||||
)
|
||||
|
||||
// UnitBound is the unit level's one bound: its running time, five seconds for the level, no setup (ADR 0445 §3). The
|
||||
// level's time is that of its slowest package, as packages run side by side, the compile not counted.
|
||||
const UnitBound = 5 * time.Second
|
||||
|
||||
// Bounds are the bounds a level is judged by: Setup is the time of everything before the tests run (compiling,
|
||||
// starting the store and the bus; the unit level has none) and Run the tests' own time.
|
||||
type Bounds struct {
|
||||
Setup, Run time.Duration
|
||||
}
|
||||
|
||||
// PackageResult is one package's part of a level.
|
||||
type PackageResult struct {
|
||||
Dir string `json:"dir"`
|
||||
Package string `json:"package"`
|
||||
Passed bool `json:"passed"`
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
Tests int `json:"tests"`
|
||||
// NoTests is a package with no test files (at this level: a file with the other level's build tag is none).
|
||||
NoTests bool `json:"no_tests,omitempty"`
|
||||
}
|
||||
|
||||
// TestTime is one test and the time it took.
|
||||
type TestTime struct {
|
||||
Package string `json:"package"`
|
||||
Test string `json:"test"`
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
}
|
||||
|
||||
// Failure is a test that failed, or a package that failed with no test to name (a build that failed, a TestMain that
|
||||
// panicked); Output is the first lines it wrote.
|
||||
type Failure struct {
|
||||
Package string `json:"package"`
|
||||
Test string `json:"test,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
}
|
||||
|
||||
// TestResult is what a test step passes with.
|
||||
type TestResult struct {
|
||||
Level string `json:"level"`
|
||||
Passed bool `json:"passed"`
|
||||
// Why says, in words, why the level failed, or that it has no tests.
|
||||
Why string `json:"why,omitempty"`
|
||||
NoTests bool `json:"no_tests,omitempty"`
|
||||
Tests int `json:"tests"`
|
||||
Skipped int `json:"skipped"`
|
||||
Packages []PackageResult `json:"packages"`
|
||||
// Elapsed is the level's running time in seconds: the slowest package's.
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
// Setup is the level's setup time in seconds, when it has a setup.
|
||||
Setup float64 `json:"setup,omitempty"`
|
||||
// Bound and SetupBound are the bounds the level was judged by, in seconds.
|
||||
Bound float64 `json:"bound"`
|
||||
SetupBound float64 `json:"setup_bound,omitempty"`
|
||||
Slowest []TestTime `json:"slowest,omitempty"`
|
||||
Failed []Failure `json:"failed,omitempty"`
|
||||
// Store and Bus are what the integration level ran against: the postgres major version and the nats-server
|
||||
// version (ADR 0470 §4).
|
||||
// OverBound says a bound was kept no longer, and OverWhich which: setup, run or both. The level's verdict, Passed, is
|
||||
// the tests' own: the checker judges the bounds (ADR 0453 §9), with the slowest tests this result names.
|
||||
OverBound bool `json:"over_bound"`
|
||||
OverWhich string `json:"over_which,omitempty"`
|
||||
Store string `json:"store,omitempty"`
|
||||
Bus string `json:"bus,omitempty"`
|
||||
}
|
||||
|
||||
// event is one line of `go test -json` (test2json, and the build events go test adds).
|
||||
type event struct {
|
||||
Action string
|
||||
Package string
|
||||
Test string
|
||||
Output string
|
||||
ImportPath string
|
||||
FailedBuild string
|
||||
Elapsed float64
|
||||
}
|
||||
|
||||
// outputLines is how many lines of output a failure keeps.
|
||||
const outputLines = 15
|
||||
|
||||
// reader collects the events of one go.mod directory's `go test -json` into a result.
|
||||
type reader struct {
|
||||
res *TestResult
|
||||
dir string
|
||||
pkgs map[string]*PackageResult
|
||||
order []string
|
||||
builds map[string][]string // build output by import path
|
||||
pkgOut map[string][]string
|
||||
testOut map[string][]string // by package + "\x00" + test
|
||||
times []TestTime
|
||||
failedIn map[string]bool // packages with a failed test
|
||||
}
|
||||
|
||||
func newReader(res *TestResult, dir string) *reader {
|
||||
return &reader{res: res, dir: dir, pkgs: map[string]*PackageResult{}, builds: map[string][]string{},
|
||||
pkgOut: map[string][]string{}, testOut: map[string][]string{}, failedIn: map[string]bool{}}
|
||||
}
|
||||
|
||||
// read takes go test's output. Lines that are not events are kept for a failure that has nothing else to say.
|
||||
func (r *reader) read(out string) (noise []string) {
|
||||
sc := bufio.NewScanner(strings.NewReader(out))
|
||||
sc.Buffer(make([]byte, 1<<20), 1<<26)
|
||||
for sc.Scan() {
|
||||
var ev event
|
||||
if err := json.Unmarshal(sc.Bytes(), &ev); err != nil {
|
||||
if l := strings.TrimSpace(sc.Text()); l != "" {
|
||||
noise = append(noise, l)
|
||||
}
|
||||
continue
|
||||
}
|
||||
r.event(ev)
|
||||
}
|
||||
return noise
|
||||
}
|
||||
|
||||
func (r *reader) pkg(name string) *PackageResult {
|
||||
p := r.pkgs[name]
|
||||
if p == nil {
|
||||
p = &PackageResult{Dir: r.dir, Package: name}
|
||||
r.pkgs[name] = p
|
||||
r.order = append(r.order, name)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func keep(lines []string, out string) []string {
|
||||
if t := strings.TrimRight(out, "\r\n"); t != "" && !strings.HasPrefix(t, "=== ") {
|
||||
lines = append(lines, t)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func (r *reader) event(ev event) {
|
||||
switch {
|
||||
case ev.Package == "" && ev.ImportPath != "":
|
||||
if ev.Action == "build-output" {
|
||||
r.builds[ev.ImportPath] = keep(r.builds[ev.ImportPath], ev.Output)
|
||||
}
|
||||
case ev.Package == "":
|
||||
case ev.Test == "":
|
||||
p := r.pkg(ev.Package)
|
||||
switch ev.Action {
|
||||
case "output":
|
||||
r.pkgOut[ev.Package] = keep(r.pkgOut[ev.Package], ev.Output)
|
||||
case "pass":
|
||||
p.Passed, p.Elapsed = true, ev.Elapsed
|
||||
case "fail":
|
||||
p.Passed, p.Elapsed = false, ev.Elapsed
|
||||
if ev.FailedBuild != "" {
|
||||
r.pkgOut[ev.Package] = append(r.builds[ev.FailedBuild], r.pkgOut[ev.Package]...)
|
||||
}
|
||||
case "skip":
|
||||
p.Passed, p.NoTests = true, true
|
||||
}
|
||||
default:
|
||||
key := ev.Package + "\x00" + ev.Test
|
||||
switch ev.Action {
|
||||
case "output":
|
||||
r.testOut[key] = keep(r.testOut[key], ev.Output)
|
||||
case "pass", "fail", "skip":
|
||||
p := r.pkg(ev.Package)
|
||||
p.Tests++
|
||||
r.res.Tests++
|
||||
if ev.Action == "skip" {
|
||||
r.res.Skipped++
|
||||
}
|
||||
r.times = append(r.times, TestTime{Package: ev.Package, Test: ev.Test, Elapsed: ev.Elapsed})
|
||||
if ev.Action == "fail" {
|
||||
r.failedIn[ev.Package] = true
|
||||
r.res.Failed = append(r.res.Failed, Failure{Package: ev.Package, Test: ev.Test, Output: clip(r.testOut[key])})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func clip(lines []string) string {
|
||||
if len(lines) > outputLines {
|
||||
lines = append(lines[:outputLines:outputLines], fmt.Sprintf("... (%d more lines)", len(lines)-outputLines))
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// finish moves what was read into the result: the packages in order, a failure for each failed package that has no
|
||||
// failed test to name it, and the tests' times.
|
||||
func (r *reader) finish() {
|
||||
for _, name := range r.order {
|
||||
p := r.pkgs[name]
|
||||
if !p.Passed && !r.failedIn[name] {
|
||||
r.res.Failed = append(r.res.Failed, Failure{Package: name, Output: clip(r.pkgOut[name])})
|
||||
}
|
||||
if !p.NoTests && p.Elapsed > r.res.Elapsed {
|
||||
r.res.Elapsed = p.Elapsed
|
||||
}
|
||||
r.res.Packages = append(r.res.Packages, *p)
|
||||
}
|
||||
r.res.Slowest = append(r.res.Slowest, r.times...)
|
||||
}
|
||||
|
||||
// judge sets Passed and Why from what was read and the bounds. A level with no failure, over no bound, passes; with no
|
||||
// test at all it says the repository has none at this level, which is neither a failure nor a warning (ADR 0445 §1).
|
||||
func (res *TestResult) judge(b Bounds) {
|
||||
sort.SliceStable(res.Slowest, func(i, j int) bool { return res.Slowest[i].Elapsed > res.Slowest[j].Elapsed })
|
||||
if len(res.Slowest) > 5 {
|
||||
res.Slowest = res.Slowest[:5]
|
||||
}
|
||||
res.Bound, res.SetupBound = b.Run.Seconds(), b.Setup.Seconds()
|
||||
var why, over []string
|
||||
if n := len(res.Failed); n > 0 {
|
||||
names := []string{}
|
||||
for _, f := range res.Failed {
|
||||
names = append(names, strings.TrimSpace(f.Package+" "+f.Test))
|
||||
}
|
||||
why = append(why, fmt.Sprintf("%d failed: %s", n, strings.Join(names, ", ")))
|
||||
}
|
||||
res.Passed = len(res.Failed) == 0
|
||||
if b.Setup > 0 && res.Setup > b.Setup.Seconds() {
|
||||
over = append(over, "setup")
|
||||
why = append(why, fmt.Sprintf("the %s level's setup took %.2f s, over its setup bound of %s", res.Level, res.Setup, seconds(b.Setup)))
|
||||
}
|
||||
if b.Run > 0 && res.Elapsed > b.Run.Seconds() {
|
||||
over = append(over, "run")
|
||||
slow := []string{}
|
||||
for _, s := range res.Slowest {
|
||||
slow = append(slow, fmt.Sprintf("%s %s (%.2f s)", s.Package, s.Test, s.Elapsed))
|
||||
}
|
||||
why = append(why, fmt.Sprintf("the %s level took %.2f s, over its bound of %s; the slowest tests: %s", res.Level, res.Elapsed, seconds(b.Run), strings.Join(slow, ", ")))
|
||||
}
|
||||
res.OverBound, res.OverWhich = len(over) > 0, strings.Join(over, ",")
|
||||
res.NoTests = res.Tests == 0 && len(res.Failed) == 0
|
||||
switch {
|
||||
case len(why) > 0:
|
||||
res.Why = strings.Join(why, "; ")
|
||||
case res.NoTests:
|
||||
res.Why = "the repository has no tests at the " + res.Level + " level"
|
||||
}
|
||||
}
|
||||
|
||||
// seconds is a duration as seconds, without trailing zeros: "5 s", "0.01 s".
|
||||
func seconds(d time.Duration) string {
|
||||
return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.3f", d.Seconds()), "0"), ".") + " s"
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
// mesh-build-go: the Go build module (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470; design 47 R4c-3, issue
|
||||
// 1483). A Go tools bundle the node's runtime launches as the module's own account (runs-as). It names the one Go
|
||||
// toolchain version it builds with, writes the version it finds to <node>.go of its `toolchains` state at start and
|
||||
// before every step, and refuses every step on a node that has another: its health is then unhealthy, so the
|
||||
// controller raises the node's condition. What it says, it says on stderr; stdout is the MCP channel.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The directories the module's manifest places outside its home (ADR 0466 §1): the workspaces, and the Go module
|
||||
// and build caches.
|
||||
const (
|
||||
WorkDir = "/var/lib/mesh-build-go-work"
|
||||
CacheDir = "/var/lib/mesh-build-go-cache"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-go] "+format+"\n", a...) }
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == busArg {
|
||||
os.Exit(runBus(os.Args[2:]))
|
||||
}
|
||||
ctx := context.Background()
|
||||
m := &Module{
|
||||
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
|
||||
Recipe: Current(),
|
||||
Probe: ProbeGo,
|
||||
Toolchains: stdio.State(delivery.ToolchainsState).Put,
|
||||
Emit: func(t string, b any) error { return stdio.Emit(t, b) },
|
||||
Steps: stdio.State(delivery.StepsState).Put,
|
||||
Ctx: ctx,
|
||||
Say: logf,
|
||||
Work: WorkDir,
|
||||
Cache: CacheDir,
|
||||
}
|
||||
if u, err := user.Current(); err == nil {
|
||||
m.Home = u.HomeDir
|
||||
}
|
||||
m.Run = StepRunner(m.Home)
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
}
|
||||
go startKeeper(ctx)
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The catalogue's root and this module's directory.
|
||||
const (
|
||||
catalogueRoot = "../../../.."
|
||||
moduleDir = "../.."
|
||||
)
|
||||
|
||||
func readManifest(t *testing.T) catalogue.Manifest {
|
||||
t.Helper()
|
||||
m, err := catalogue.ReadTreeManifest(catalogueRoot, moduleDir)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's validator refuses module.json: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func resource(t *testing.T, m catalogue.Manifest, id string) map[string]any {
|
||||
t.Helper()
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("module.json declares no resource %q", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1483/AC1: mesh-sdk's validator accepts the module (alone on a shelf too), and it runs as mesh-build-go with its
|
||||
// home at mode 0700, requires user-namespaces and declares the four packages.
|
||||
func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if problems := catalogue.CatalogueProblems(catalogue.Shelf{m.Module: m}); len(problems) > 0 {
|
||||
t.Errorf("the catalogue's rules refuse it: %v", problems)
|
||||
}
|
||||
if m.Module != "mesh-build-go" || m.RunsAs != "mesh-build-go" {
|
||||
t.Errorf("module %q runs as %q, want mesh-build-go as itself", m.Module, m.RunsAs)
|
||||
}
|
||||
account := resource(t, m, "account")
|
||||
if account["type"] != "user" || account["name"] != "mesh-build-go" || account["shell"] != "/usr/bin/nologin" || account["home"] != "/var/lib/mesh-build-go" {
|
||||
t.Errorf("account = %v", account)
|
||||
}
|
||||
home := resource(t, m, "state")
|
||||
if home["type"] != "directory" || home["mode"] != "0700" || home["owner"] != "mesh-build-go" || home["place"] != "." {
|
||||
t.Errorf("home = %v, want its directory at mode 0700 owned by the account", home)
|
||||
}
|
||||
if !slices.Contains(m.Capabilities, "user-namespaces") {
|
||||
t.Errorf("capabilities = %v, want user-namespaces", m.Capabilities)
|
||||
}
|
||||
packages := []string{}
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "package" {
|
||||
packages = append(packages, r["package"].(string))
|
||||
}
|
||||
}
|
||||
slices.Sort(packages)
|
||||
if want := []string{"git", "go", "openssh", "postgresql"}; !slices.Equal(packages, want) {
|
||||
t.Errorf("packages = %v, want %v", packages, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The work and cache directories are outside the home (ADR 0466 §1), owned by the account.
|
||||
func TestWorkAndCacheDirectoriesLieOutsideTheHome(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
for _, id := range []string{"work", "cache"} {
|
||||
r := resource(t, m, id)
|
||||
path, _ := r["path"].(string)
|
||||
if path == "" || path == "/var/lib/mesh-build-go" || len(path) > len("/var/lib/mesh-build-go") && path[:len("/var/lib/mesh-build-go/")] == "/var/lib/mesh-build-go/" {
|
||||
t.Errorf("%s is at %q, inside or at the home", id, path)
|
||||
}
|
||||
if r["owner"] != "mesh-build-go" {
|
||||
t.Errorf("%s is owned by %v", id, r["owner"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The states are the three the module keeps, no seat is claimed (ADR 0469 §6), and the health is the module's own
|
||||
// tool, which the manifest also lists among its tools.
|
||||
func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
states := []string{}
|
||||
for _, s := range m.State {
|
||||
states = append(states, s.Name)
|
||||
}
|
||||
slices.Sort(states)
|
||||
if want := []string{"keys", "steps", "toolchains"}; !slices.Equal(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if len(m.Claims) != 0 {
|
||||
t.Errorf("claims = %v, want no seat", m.Claims)
|
||||
}
|
||||
if !slices.Contains(m.Tools, HealthTool) {
|
||||
t.Errorf("tools = %v lack %s", m.Tools, HealthTool)
|
||||
}
|
||||
raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json"))
|
||||
if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) {
|
||||
t.Errorf("runtime-health does not name %s", HealthTool)
|
||||
}
|
||||
}
|
||||
|
||||
// Every tool the manifest names is served, and every tool served is named.
|
||||
func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
served := []string{}
|
||||
for _, tool := range (&Module{}).Tools((&Module{}).StepTools()) {
|
||||
served = append(served, tool.Name)
|
||||
}
|
||||
slices.Sort(served)
|
||||
named := slices.Clone(m.Tools)
|
||||
slices.Sort(named)
|
||||
if !slices.Equal(served, named) {
|
||||
t.Errorf("served %v, manifest names %v", served, named)
|
||||
}
|
||||
}
|
||||
|
||||
// 1482/AC4: gitea's manifest reads the keys state of both build modules, and the catalogue's validator accepts it with the
|
||||
// build modules beside it: each owner present declares the state read (a read of a state its owner does not keep is
|
||||
// refused).
|
||||
func TestGiteaReadsTheBuildModulesKeysAndTheCatalogueAcceptsThemTogether(t *testing.T) {
|
||||
var all []catalogue.Manifest
|
||||
var gitea catalogue.Manifest
|
||||
for _, name := range []string{"gitea", "mesh-build-go", "mesh-build-node"} {
|
||||
m, err := catalogue.ReadTreeManifest(catalogueRoot, filepath.Join(catalogueRoot, "modules", name))
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's validator refuses modules/%s/module.json: %v", name, err)
|
||||
}
|
||||
all = append(all, m)
|
||||
if name == "gitea" {
|
||||
gitea = m
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"mesh-build-go.keys", "mesh-build-node.keys", "ssh-client.keys"} {
|
||||
if !slices.Contains(gitea.Reads, want) {
|
||||
t.Errorf("gitea reads %v, lacks %s", gitea.Reads, want)
|
||||
}
|
||||
}
|
||||
if problems := catalogue.StateReadsNothingDeclares(all); len(problems) > 0 {
|
||||
t.Errorf("a read names a state its owner does not keep: %v", problems)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// CheckRound is how often the module reads the installed toolchain again, beside the check before every step and
|
||||
// the one every health look makes.
|
||||
const CheckRound = 10 * time.Minute
|
||||
|
||||
// Module is the Go build module's one running state: the node it runs on, its recipe, the toolchain guard and the
|
||||
// states it writes (toolchains, steps).
|
||||
type Module struct {
|
||||
Node string
|
||||
Recipe Recipe
|
||||
// Probe reads the installed Go version (ProbeGo).
|
||||
Probe func(ctx context.Context) (string, error)
|
||||
// Toolchains and Steps are the Put of the states of those names.
|
||||
Toolchains, Steps buildstep.Put
|
||||
// Emit tells the bus an event (stdio.Emit); nil emits nothing. The guard emits toolchain.reported after each write of
|
||||
// the toolchains key, so the indexer, which holds no timer, learns that the bucket exists (issue 1444).
|
||||
Emit func(eventType string, body any) error
|
||||
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
|
||||
Ctx context.Context
|
||||
Say func(format string, a ...any)
|
||||
// Run runs the commands of a step (StepRunner); Work holds the workspaces and Cache the module and build caches,
|
||||
// both outside the account's home (ADR 0466 §1); Home is the account's home.
|
||||
Run forgekey.Runner
|
||||
Home, Work, Cache string
|
||||
// UnitBound is the unit level's bound; zero is UnitBound. A test lowers it to see a level over its bound.
|
||||
UnitBound time.Duration
|
||||
// IntegrationBounds are the integration level's bounds; zero values are IntegrationSetup and IntegrationRun.
|
||||
IntegrationBounds Bounds
|
||||
// Now is the clock the levels' times are read from; time.Now when nil.
|
||||
Now func() time.Time
|
||||
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
|
||||
// program, StartBus in bus.go, when nil).
|
||||
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
|
||||
}
|
||||
|
||||
func (m *Module) guard() buildstep.Guard {
|
||||
return buildstep.Guard{Language: "go", Want: m.Recipe.Go, Recipe: m.Recipe.Version(), Probe: m.Probe, Node: m.Node, Put: m.reportingPut()}
|
||||
}
|
||||
|
||||
// Check reads the installed toolchain, writes <node>.go of the toolchains state and answers nil, or the refusal
|
||||
// (buildstep.Guard.Check). It runs at start, before every step, at every health look and every CheckRound.
|
||||
func (m *Module) Check(ctx context.Context) error {
|
||||
return m.guard().Check(ctx)
|
||||
}
|
||||
|
||||
// Start checks once, as the module starts, and then every CheckRound until ctx ends.
|
||||
func (m *Module) Start(ctx context.Context) {
|
||||
for {
|
||||
if err := m.Check(ctx); err != nil && m.Say != nil {
|
||||
m.Say("the toolchain: %v", err)
|
||||
}
|
||||
m.CleanWorkspaces(time.Now())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(CheckRound):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy while the toolchain installed is not
|
||||
// the recipe's, saying both versions, so the controller raises the node's condition. It looks again each time it is
|
||||
// asked.
|
||||
func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
err := m.Check(ctx)
|
||||
switch {
|
||||
case err == nil:
|
||||
return health.Healthy(fmt.Sprintf("%s installed, as the recipe %s builds with", m.Recipe.Go, m.Recipe.Version()))
|
||||
default:
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on
|
||||
// its key in the steps state. The guard runs inside the step, before run, so a node that refuses ends every step
|
||||
// refused with both versions named.
|
||||
func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) {
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return run(ctx, args)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, _ := delivery.StepKey(m.Node, id)
|
||||
return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error.
|
||||
func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
if err := m.Check(m.Ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return run(m.Ctx, args)
|
||||
}
|
||||
}
|
||||
|
||||
// Status is what the module reports of itself: the node, the recipe, the toolchain found and why it refuses, if it
|
||||
// does.
|
||||
func (m *Module) Status(ctx context.Context) map[string]any {
|
||||
err := m.Check(ctx)
|
||||
installed, _ := m.Probe(ctx)
|
||||
out := map[string]any{
|
||||
"node": m.Node, "recipe": m.Recipe.Version(), "go": m.Recipe.Go, "installed": installed,
|
||||
"postgres-major": m.Recipe.PostgresMajor, "nats-server": m.Recipe.NatsServer, "refusing": err != nil,
|
||||
}
|
||||
if err != nil {
|
||||
out["why"] = err.Error()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ToolchainReported is the event the guard emits after it writes the toolchains key.
|
||||
const ToolchainReported = "toolchain.reported"
|
||||
|
||||
// reportingPut is the toolchains Put that then emits toolchain.reported with the node and the language. A write that
|
||||
// failed emits nothing; an emit that failed is said and does not refuse the step.
|
||||
func (m *Module) reportingPut() buildstep.Put {
|
||||
return func(key string, value any) (uint64, error) {
|
||||
rev, err := m.Toolchains(key, value)
|
||||
if err != nil || m.Emit == nil {
|
||||
return rev, err
|
||||
}
|
||||
if eerr := m.Emit(ToolchainReported, map[string]string{"node": m.Node, "language": "go"}); eerr != nil && m.Say != nil {
|
||||
m.Say("the event %s could not be emitted: %v", ToolchainReported, eerr)
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// keptStates stands in for the toolchains and the steps states.
|
||||
type keptStates struct {
|
||||
mu sync.Mutex
|
||||
toolchains map[string]delivery.Toolchain
|
||||
steps map[string][]delivery.Step
|
||||
ended chan string
|
||||
}
|
||||
|
||||
func newStates() *keptStates {
|
||||
return &keptStates{toolchains: map[string]delivery.Toolchain{}, steps: map[string][]delivery.Step{}, ended: make(chan string, 16)}
|
||||
}
|
||||
|
||||
func (s *keptStates) putToolchain(key string, value any) (uint64, error) {
|
||||
tc, err := delivery.DecodeToolchain(value.(json.RawMessage))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.toolchains[key] = tc
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (s *keptStates) putStep(key string, value any) (uint64, error) {
|
||||
st, err := delivery.DecodeStep(value.(json.RawMessage))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.steps[key] = append(s.steps[key], st)
|
||||
s.mu.Unlock()
|
||||
if st.State != delivery.StepRunning {
|
||||
s.ended <- key
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (s *keptStates) toolchain(key string) (delivery.Toolchain, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
tc, ok := s.toolchains[key]
|
||||
return tc, ok
|
||||
}
|
||||
|
||||
func (s *keptStates) waitEnd(t *testing.T) delivery.Step {
|
||||
t.Helper()
|
||||
select {
|
||||
case key := <-s.ended:
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v := s.steps[key]
|
||||
return v[len(v)-1]
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the step never ended on its key")
|
||||
return delivery.Step{}
|
||||
}
|
||||
}
|
||||
|
||||
func moduleWith(states *keptStates, installed string) *Module {
|
||||
return &Module{Node: "ace", Recipe: Recipe{Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"},
|
||||
Probe: func(context.Context) (string, error) { return installed, nil },
|
||||
Toolchains: states.putToolchain, Steps: states.putStep, Ctx: context.Background()}
|
||||
}
|
||||
|
||||
// 1483/AC2: a probe answering go1.27.1: the bundle's start writes ace.go with that version and the recipe's, and
|
||||
// the health tool answers healthy.
|
||||
func TestStartWritesTheToolchainAndHealthIsHealthy(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go m.Start(ctx)
|
||||
var tc delivery.Toolchain
|
||||
for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
|
||||
if got, ok := states.toolchain("ace.go"); ok {
|
||||
tc = got
|
||||
break
|
||||
}
|
||||
}
|
||||
if tc.Version != "go1.27.1" || tc.Recipe != m.Recipe.Version() || tc.Language != "go" {
|
||||
t.Fatalf("ace.go = %+v, want go1.27.1 and recipe %s", tc, m.Recipe.Version())
|
||||
}
|
||||
if a := m.Health(ctx); a.State != health.StateHealthy || !a.Healthy {
|
||||
t.Errorf("health = %+v, want healthy", a)
|
||||
}
|
||||
}
|
||||
|
||||
// 1483/AC3: a probe answering go1.27.2: every step tool's step ends refused naming both versions, ace.go holds
|
||||
// go1.27.2, and the health tool answers unhealthy naming both.
|
||||
func TestAnotherGoRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.2")
|
||||
ran := false
|
||||
run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
|
||||
steps := append([]StepTool{
|
||||
{Name: "long_probe", Long: true, Run: run},
|
||||
{Name: "short_probe", Run: run},
|
||||
}, m.StepTools()...)
|
||||
for _, tool := range m.Tools(steps) {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool {
|
||||
continue
|
||||
}
|
||||
var reason string
|
||||
out, err := tool.Run(map[string]any{})
|
||||
if isLong(steps, tool.Name) {
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tool.Name, err)
|
||||
}
|
||||
if out.(map[string]any)["step"] == "" {
|
||||
t.Fatalf("%s answered no step id: %v", tool.Name, out)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepRefused {
|
||||
t.Errorf("%s: the step ended %s, want refused", tool.Name, st.State)
|
||||
}
|
||||
reason = st.Reason
|
||||
} else {
|
||||
if err == nil {
|
||||
t.Fatalf("%s: answered %v, want a refusal", tool.Name, out)
|
||||
}
|
||||
reason = err.Error()
|
||||
}
|
||||
if !strings.Contains(reason, "go1.27.2") || !strings.Contains(reason, "go1.27.1") {
|
||||
t.Errorf("%s: the refusal %q does not name both versions", tool.Name, reason)
|
||||
}
|
||||
}
|
||||
if ran {
|
||||
t.Errorf("a step ran on a node that refuses")
|
||||
}
|
||||
if tc, _ := states.toolchain("ace.go"); tc.Version != "go1.27.2" {
|
||||
t.Errorf("ace.go holds %q, want the installed go1.27.2", tc.Version)
|
||||
}
|
||||
a := m.Health(context.Background())
|
||||
if a.State != health.StateUnhealthy || !strings.Contains(a.Why, "go1.27.2") || !strings.Contains(a.Why, "go1.27.1") {
|
||||
t.Errorf("health = %+v, want unhealthy naming both versions", a)
|
||||
}
|
||||
}
|
||||
|
||||
func isLong(steps []StepTool, name string) bool {
|
||||
for _, s := range steps {
|
||||
if s.Name == name {
|
||||
return s.Long
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// 1483/AC4: recipes differing only in the postgres major or the nats-server version have different recipe versions;
|
||||
// equal recipes have the same one.
|
||||
func TestRecipeVersionMovesWithEachOfItsValues(t *testing.T) {
|
||||
base := Recipe{Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"}
|
||||
if base.Version() != (Recipe{Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"}).Version() {
|
||||
t.Errorf("one recipe has two versions")
|
||||
}
|
||||
seen := map[string]string{base.Version(): "base"}
|
||||
for name, r := range map[string]Recipe{
|
||||
"postgres major": {Steps: "1", Go: "go1.27.1", PostgresMajor: 19, NatsServer: "2.11.17"},
|
||||
"nats-server": {Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.18"},
|
||||
"go": {Steps: "1", Go: "go1.27.2", PostgresMajor: 18, NatsServer: "2.11.17"},
|
||||
"steps": {Steps: "2", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"},
|
||||
} {
|
||||
v := r.Version()
|
||||
if was, dup := seen[v]; dup {
|
||||
t.Errorf("the recipe differing in %s has the version of the %s recipe: %s", name, was, v)
|
||||
}
|
||||
seen[v] = name
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRecipeNamesGo1271AndPostgres18AndTheLibrarysNatsVersion(t *testing.T) {
|
||||
r := Current()
|
||||
if r.Go != "go1.27.1" || r.PostgresMajor != 18 {
|
||||
t.Errorf("recipe = %+v", r)
|
||||
}
|
||||
if r.NatsServer == "" || strings.HasPrefix(r.NatsServer, "v") {
|
||||
t.Errorf("nats-server version = %q", r.NatsServer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGoVersion(t *testing.T) {
|
||||
if v, err := parseGoVersion("go version go1.27.1 linux/amd64\n"); err != nil || v != "go1.27.1" {
|
||||
t.Errorf("= %q, %v", v, err)
|
||||
}
|
||||
if _, err := parseGoVersion("not go"); err == nil {
|
||||
t.Errorf("accepted a text that is not `go version`'s")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProbeTimeout bounds `go version`.
|
||||
const ProbeTimeout = 15 * time.Second
|
||||
|
||||
// ProbeGo reads the installed Go version: the third word `go version` prints ("go version go1.27.1 linux/amd64"),
|
||||
// as the recipe's RecipeGo is written. GOTOOLCHAIN=local keeps an older Go from fetching another toolchain: what is
|
||||
// installed is what is read.
|
||||
func ProbeGo(ctx context.Context) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, ProbeTimeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "go", "version")
|
||||
cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return parseGoVersion(string(out))
|
||||
}
|
||||
|
||||
func parseGoVersion(out string) (string, error) {
|
||||
f := strings.Fields(out)
|
||||
if len(f) < 3 || f[0] != "go" || f[1] != "version" || !strings.HasPrefix(f[2], "go") {
|
||||
return "", fmt.Errorf("`go version` answered %q", strings.TrimSpace(out))
|
||||
}
|
||||
return f[2], nil
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
// The recipe of the Go build module (novox/hq ADR 0405 §1 and §7, ADR 0469 §1, ADR 0470 §4; issue 1483): the one
|
||||
// Go toolchain version it builds with, the one postgres major version its integration level tests with, and the
|
||||
// recipe's own version. A node whose installed Go differs from the recipe's refuses every step (guard.go), so all
|
||||
// builders run this version or none builds.
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
)
|
||||
|
||||
// RecipeGo is the Go version the recipe builds with: the text `go version` prints for it, which is what the guard
|
||||
// compares and what the indexer reads from <node>.go of the toolchains state.
|
||||
const RecipeGo = "go1.27.1"
|
||||
|
||||
// PostgresMajor is the postgres major version the integration level tests with (ADR 0470 §4).
|
||||
const PostgresMajor = 18
|
||||
|
||||
// RecipeSteps is the recipe's own version: changed whenever the steps of fetch, test or build change in a way
|
||||
// that can change what they produce (ADR 0405 §7). It is one of the four values the recipe version hashes.
|
||||
const RecipeSteps = "1"
|
||||
|
||||
// natsModule is the library the integration level's bus is started from (ADR 0470 §2).
|
||||
const natsModule = "github.com/nats-io/nats-server/v2"
|
||||
|
||||
// Recipe is every value that makes up the recipe version.
|
||||
type Recipe struct {
|
||||
// Steps is RecipeSteps.
|
||||
Steps string
|
||||
// Go is the toolchain's version as `go version` prints it.
|
||||
Go string
|
||||
// PostgresMajor is the integration level's postgres major version.
|
||||
PostgresMajor int
|
||||
// NatsServer is the nats-server library's version, from the program's build information.
|
||||
NatsServer string
|
||||
}
|
||||
|
||||
// Current is the recipe of this build of the module.
|
||||
func Current() Recipe {
|
||||
return Recipe{Steps: RecipeSteps, Go: RecipeGo, PostgresMajor: PostgresMajor, NatsServer: natsServerVersion()}
|
||||
}
|
||||
|
||||
// Version is the recipe version: a hash of the recipe's own version, the Go version, the postgres major version and
|
||||
// the nats-server version, so a change of any of the four moves it and with it every fingerprint that holds it
|
||||
// (ADR 0405 §7, ADR 0465 §2). The fields are named in the hashed text, so no two recipes share a text.
|
||||
func (r Recipe) Version() string {
|
||||
h := sha256.New()
|
||||
for _, f := range [][2]string{
|
||||
{"steps", r.Steps}, {"go", r.Go}, {"postgres", strconv.Itoa(r.PostgresMajor)}, {"nats-server", r.NatsServer},
|
||||
} {
|
||||
h.Write([]byte(f[0] + "=" + f[1] + "\n"))
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))[:16]
|
||||
}
|
||||
|
||||
// natsServerVersion is the version of the nats-server library this program was built with: its build information
|
||||
// names it as the module's go.mod does, and has no second source (ADR 0470 §4). A binary built without build
|
||||
// information (a stripped test binary) falls back to the library's own version constant, which is the same text.
|
||||
func natsServerVersion() string {
|
||||
if info, ok := debug.ReadBuildInfo(); ok {
|
||||
if v := natsVersionOf(info); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return server.VERSION
|
||||
}
|
||||
|
||||
// natsVersionOf is the version build information gives the nats-server library, "v" stripped, or "" when it lists
|
||||
// none. A replaced library is the replacement's version.
|
||||
func natsVersionOf(info *debug.BuildInfo) string {
|
||||
for _, d := range info.Deps {
|
||||
if d.Path != natsModule {
|
||||
continue
|
||||
}
|
||||
if d.Replace != nil {
|
||||
d = d.Replace
|
||||
}
|
||||
return strings.TrimPrefix(d.Version, "v")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// StepRunner runs the commands of a build step with an environment built from scratch (novox/hq ADR 0466 §4, issue
|
||||
// 1485): PATH, HOME (the account's own home, where its ssh files and git configuration are) and a fixed locale, plus
|
||||
// the words the caller passes, and nothing else of this process's environment — which holds the module's bus
|
||||
// credentials and might hold a password or a token. It is mesh-sdk's clean runner (ExecRunnerClean, issue 1497) with
|
||||
// that base given; a command is bound by its context alone (the bound 0), so a fetch may outlast a tool call's thirty
|
||||
// seconds (the step has its own context).
|
||||
func StepRunner(home string) forgekey.Runner {
|
||||
run := forgekey.ExecRunnerClean(0)
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
return run(ctx, append(baseEnv(home), env...), name, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// defaultPath is where a step looks for git, ssh and go when this process was given no PATH.
|
||||
const defaultPath = "/usr/local/bin:/usr/bin:/bin"
|
||||
|
||||
func baseEnv(home string) []string {
|
||||
path := os.Getenv("PATH")
|
||||
if path == "" {
|
||||
path = defaultPath
|
||||
}
|
||||
return []string{"PATH=" + path, "HOME=" + home, "LC_ALL=C"}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
// The integration level's throwaway store (novox/hq ADR 0470 §1; issue 1486): one postgres server of the installed
|
||||
// package for each run, `initdb` into the run's own directory, listening on a unix socket in that directory and on no
|
||||
// TCP address, as the module's account, with durability off as today's check sets it.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// storeStopBound is how long stopping the store is given: it runs after the level, whatever the level did.
|
||||
const storeStopBound = 30 * time.Second
|
||||
|
||||
var postgresVersion = regexp.MustCompile(`PostgreSQL\)\s+(\d+)`)
|
||||
|
||||
// postgresMajor is the major version of the installed postgres: `postgres --version` ("postgres (PostgreSQL) 18.6").
|
||||
func (m *Module) postgresMajor(ctx context.Context) (int, error) {
|
||||
r := m.Run(ctx, nil, "postgres", "--version")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return 0, fmt.Errorf("`postgres --version`: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
f := postgresVersion.FindStringSubmatch(r.Stdout)
|
||||
if f == nil {
|
||||
return 0, fmt.Errorf("`postgres --version` answered %q", firstLine(r.Stdout))
|
||||
}
|
||||
return strconv.Atoi(f[1])
|
||||
}
|
||||
|
||||
// checkPostgres refuses the integration level when the installed postgres major is not the recipe's, naming both
|
||||
// (ADR 0470 §4); the unit level is not refused. It answers the major it found.
|
||||
func (m *Module) checkPostgres(ctx context.Context) (int, error) {
|
||||
major, err := m.postgresMajor(ctx)
|
||||
if err != nil {
|
||||
return 0, &buildstep.Refusal{Text: fmt.Sprintf("the installed postgres version could not be read: %v", err), Want: strconv.Itoa(m.Recipe.PostgresMajor)}
|
||||
}
|
||||
if major != m.Recipe.PostgresMajor {
|
||||
return major, &buildstep.Refusal{Text: fmt.Sprintf("postgres %d installed, the integration level is built for postgres %d", major, m.Recipe.PostgresMajor),
|
||||
Want: strconv.Itoa(m.Recipe.PostgresMajor), Have: strconv.Itoa(major)}
|
||||
}
|
||||
return major, nil
|
||||
}
|
||||
|
||||
// socketLimit is the longest path of a socket postgres accepts, with the file's name: it refuses 108 bytes and over.
|
||||
const socketLimit = 100
|
||||
|
||||
// socketName is the file postgres makes in the socket directory.
|
||||
const socketName = "/.s.PGSQL.5432"
|
||||
|
||||
// socketDir makes the directory of the store's socket: under the run when the socket's path stays under socketLimit,
|
||||
// else a short directory of its own under /tmp (answering short, so the caller removes it). A unix socket's path is
|
||||
// bounded far below a file's, and a work directory deep in a data root is over it.
|
||||
func socketDir(run string) (dir string, short bool, err error) {
|
||||
dir = filepath.Join(run, "pg", "socket")
|
||||
if len(dir)+len(socketName) < socketLimit {
|
||||
return dir, false, os.MkdirAll(dir, 0o700)
|
||||
}
|
||||
dir, err = os.MkdirTemp("/tmp", "pg")
|
||||
return dir, true, err
|
||||
}
|
||||
|
||||
// storeSettings are the server's own settings: no TCP address, a socket directory, and no durability.
|
||||
func storeOptions(socket string) string {
|
||||
return "-k " + socket + " -c listen_addresses='' -c fsync=off -c synchronous_commit=off -c full_page_writes=off"
|
||||
}
|
||||
|
||||
// startStore makes and starts the run's postgres in run/pg and answers the address tests are given as
|
||||
// MESH_TEST_POSTGRES, and what stops it. The address names the socket directory percent-encoded, as the controller's
|
||||
// tests cut the URL at its last slash to name their own databases.
|
||||
func (m *Module) startStore(ctx context.Context, run string) (string, func(), error) {
|
||||
data := filepath.Join(run, "pg", "data")
|
||||
socket, short, err := socketDir(run)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
stop := func() {
|
||||
if short {
|
||||
defer os.RemoveAll(socket)
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(context.Background(), storeStopBound)
|
||||
defer cancel()
|
||||
m.Run(sctx, nil, "pg_ctl", "-D", data, "-m", "immediate", "-w", "stop")
|
||||
}
|
||||
if r := m.Run(ctx, nil, "initdb", "-D", data, "-U", "postgres", "--auth=trust", "-E", "UTF8", "--no-locale"); r.Status != 0 || r.Err != "" {
|
||||
return "", nil, fmt.Errorf("initdb: %s", firstLine(r.Err+" "+r.Stderr+" "+r.Stdout))
|
||||
}
|
||||
if r := m.Run(ctx, nil, "pg_ctl", "-D", data, "-l", filepath.Join(run, "pg", "log"), "-w", "-t", "30", "-o", storeOptions(socket), "start"); r.Status != 0 || r.Err != "" {
|
||||
stop() // a server that was half started is stopped
|
||||
return "", nil, fmt.Errorf("pg_ctl start: %s", firstLine(r.Err+" "+r.Stderr+" "+r.Stdout))
|
||||
}
|
||||
return "postgres://postgres@/postgres?host=" + url.QueryEscape(socket) + "&sslmode=disable", stop, nil
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package main
|
||||
|
||||
// The test step (novox/hq ADR 0445; issues 1290 and 1486): a fetched workspace's tests by Go's convention, one level
|
||||
// at a time, each judged by its own bounds. The unit level is `go test` over every package without the `integration`
|
||||
// build tag.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// testTimeout is go test's own timeout for a level: a bound on a test that never ends, far above any level's bound.
|
||||
const testTimeout = "10m"
|
||||
|
||||
var packageOK = regexp.MustCompile(`^(\./|[A-Za-z0-9])[A-Za-z0-9._/@~-]*(/\.\.\.)?$`)
|
||||
|
||||
func (m *Module) testTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_go_test",
|
||||
Description: "Run the tests of a fetched workspace by Go's convention, one level: the unit level is go test over every " +
|
||||
"package without the integration build tag, within five seconds. Answers a step id at once; the step ends on " +
|
||||
"its key with the level's result: its packages, its times, the tests that failed and the slowest tests. A " +
|
||||
"level that fails is a finished step whose result says passed false.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
|
||||
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
|
||||
"packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "package patterns of each module, ./... when none"},
|
||||
},
|
||||
"required": []string{"workspace", "level"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.test,
|
||||
}
|
||||
}
|
||||
|
||||
// unitBound is the unit level's bound: UnitBound unless a test lowered it.
|
||||
func (m *Module) unitBound() time.Duration {
|
||||
if m.UnitBound > 0 {
|
||||
return m.UnitBound
|
||||
}
|
||||
return UnitBound
|
||||
}
|
||||
|
||||
func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
level, _ := args["level"].(string)
|
||||
packages, err := packagesArg(args["packages"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
tree := filepath.Join(m.Work, workspace, "tree")
|
||||
if _, err := os.Stat(tree); err != nil {
|
||||
return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
|
||||
}
|
||||
switch level {
|
||||
case LevelUnit:
|
||||
return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}})
|
||||
case LevelIntegration:
|
||||
return m.integration(ctx, workspace, tree, packages)
|
||||
}
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
}
|
||||
|
||||
func packagesArg(v any) ([]string, error) {
|
||||
if v == nil {
|
||||
return nil, nil
|
||||
}
|
||||
list, ok := v.([]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("packages is a list of package patterns")
|
||||
}
|
||||
out := []string{}
|
||||
for _, e := range list {
|
||||
s, _ := e.(string)
|
||||
if !packageOK.MatchString(s) || strings.Contains(strings.TrimSuffix(s, "/..."), "..") {
|
||||
return nil, fmt.Errorf("%q is not a package pattern", s)
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// levelOpts are what a level asks of runLevel.
|
||||
type levelOpts struct {
|
||||
level string
|
||||
tags []string
|
||||
packages []string
|
||||
bounds Bounds
|
||||
// env is what the level adds to a go command's environment (MESH_TEST_POSTGRES, MESH_TEST_NATS).
|
||||
env []string
|
||||
// setup is the time the level's setup took, which the caller measured; wall says the level's running time is the
|
||||
// wall time of its go test commands, as the integration level's packages run side by side by go test's own
|
||||
// parallelism, where the unit level's is the slowest package's.
|
||||
setup time.Duration
|
||||
wall bool
|
||||
// compose is a state of the level to put in the result (the store and the bus it ran against).
|
||||
store, bus string
|
||||
}
|
||||
|
||||
// runLevel runs `go test -json` in each go.mod directory of the tree and judges the whole against the bounds.
|
||||
func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestResult, error) {
|
||||
dirs, err := m.goModDirs(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &TestResult{Level: o.level, Packages: []PackageResult{}, Setup: o.setup.Seconds(), Store: o.store, Bus: o.bus}
|
||||
packages := o.packages
|
||||
if len(packages) == 0 {
|
||||
packages = []string{"./..."}
|
||||
}
|
||||
began := m.clock()
|
||||
for _, d := range dirs {
|
||||
args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
|
||||
if len(o.tags) > 0 {
|
||||
args = append(args, "-tags", strings.Join(o.tags, ","))
|
||||
}
|
||||
args = append(args, packages...)
|
||||
r := m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
|
||||
rd := newReader(res, d)
|
||||
noise := rd.read(r.Stdout)
|
||||
rd.finish()
|
||||
if (r.Status != 0 || r.Err != "") && len(rd.pkgs) == 0 {
|
||||
// go test said nothing a test said: it could not run (a go.mod it cannot read, a toolchain it cannot find).
|
||||
return nil, fmt.Errorf("go test in %s: %s", d, firstLine(r.Err+"\n"+strings.Join(noise, "\n")+"\n"+r.Stderr))
|
||||
}
|
||||
}
|
||||
if o.wall {
|
||||
res.Elapsed = m.clock().Sub(began).Seconds()
|
||||
}
|
||||
res.judge(o.bounds)
|
||||
return res, nil
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestMain makes this test binary the bus when the module starts it as one (bus.go).
|
||||
func TestMain(m *testing.M) {
|
||||
if len(os.Args) > 1 && os.Args[1] == busArg {
|
||||
os.Exit(runBus(os.Args[2:]))
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
// needTools skips, loudly, a test whose tools are not installed where the check runs (the mesh's Go toolchain image
|
||||
// holds no postgres): the level is proven live on a build node, which has them (issue 1486).
|
||||
func needTools(t *testing.T, what string, tools ...string) {
|
||||
t.Helper()
|
||||
for _, tool := range tools {
|
||||
if _, err := exec.LookPath(tool); err != nil {
|
||||
t.Skipf("NOT RUN: no %s installed where this check runs (%s); proven live on a build node", tool, what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// repository copies a fixture repository into a git tree, as a fetch leaves one.
|
||||
func repository(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
tree := t.TempDir()
|
||||
if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", name)+"/.", tree).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%s: %v %s", name, err, out)
|
||||
}
|
||||
for _, c := range [][]string{{"init", "-q"}, {"add", "-A"}, {"-c", "user.email=a@b", "-c", "user.name=x", "commit", "-qm", "x"}} {
|
||||
if out, err := exec.Command("git", append([]string{"-C", tree}, c...)...).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%s: git %v: %v %s", name, c, err, out)
|
||||
}
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
// Issue 1290: the unit level against the real go on the fixture repositories in testdata/repos: the tests of
|
||||
// test_test.go read the output captured from them, and this one proves the capture is what the real go says today. It
|
||||
// starts go test, so it carries the tag.
|
||||
func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) {
|
||||
needTools(t, "the unit level on fixtures", "go", "git")
|
||||
want := map[string]struct {
|
||||
passed, noTests bool
|
||||
failed string
|
||||
}{
|
||||
"failing": {failed: "fixture/failing/bad"},
|
||||
"tagged": {passed: true, noTests: true},
|
||||
"empty": {passed: true, noTests: true},
|
||||
"slow": {passed: true},
|
||||
"broken": {failed: "fixture/broken"},
|
||||
}
|
||||
for name, w := range want {
|
||||
tree := repository(t, name)
|
||||
home := t.TempDir()
|
||||
m := &Module{Home: home, Work: t.TempDir(), Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
res, err := m.runLevel(context.Background(), tree, levelOpts{level: LevelUnit, bounds: Bounds{Run: UnitBound}})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if res.Passed != w.passed || res.NoTests != w.noTests || (w.failed != "" && (len(res.Failed) != 1 || res.Failed[0].Package != w.failed)) {
|
||||
t.Errorf("%s: result = %+v", name, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processesNamed are the processes whose command line holds text.
|
||||
func processesNamed(text string) []string {
|
||||
var out []string
|
||||
cmdlines, _ := filepath.Glob("/proc/[0-9]*/cmdline")
|
||||
for _, f := range cmdlines {
|
||||
if raw, err := os.ReadFile(f); err == nil && strings.Contains(string(raw), text) {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 1486/AC1: given a fixture repository whose integration test opens MESH_TEST_POSTGRES and MESH_TEST_NATS, the
|
||||
// integration level passes, and afterwards no postgres or bus process of the run is left and its directory is gone.
|
||||
// It starts postgres and a bus, so it runs in the merge check and in a try, never on the way.
|
||||
func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.T) {
|
||||
needTools(t, "the integration level", "go", "git", "initdb", "pg_ctl", "postgres")
|
||||
tree := repository(t, "store")
|
||||
// A short work directory: a unix socket's path is bounded at 108 bytes.
|
||||
work, err := os.MkdirTemp("/tmp", "w")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(work) })
|
||||
if err := os.MkdirAll(filepath.Join(work, "ws1"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tree, filepath.Join(work, "ws1", "tree")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
home := t.TempDir()
|
||||
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the integration level: %v", err)
|
||||
}
|
||||
if !res.Passed || res.Tests != 1 || res.Setup <= 0 || res.Setup > IntegrationSetup.Seconds() {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
entries, _ := os.ReadDir(filepath.Join(work, "ws1"))
|
||||
if len(entries) != 1 || entries[0].Name() != "tree" {
|
||||
t.Errorf("the workspace holds %v, want only its tree: the run directory is gone", entries)
|
||||
}
|
||||
for deadline := time.Now().Add(5 * time.Second); len(processesNamed(work)) > 0 && time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
}
|
||||
if left := processesNamed(work); len(left) > 0 {
|
||||
t.Errorf("processes of the run are left: %v", left)
|
||||
}
|
||||
}
|
||||
|
||||
// 1487/AC2: given a fixture Go module, when it is built twice, both digests are equal and the archive holds one
|
||||
// executable named as binaryName says. It runs the real go, so it carries the tag.
|
||||
func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) {
|
||||
needTools(t, "the build", "go")
|
||||
work := t.TempDir()
|
||||
tree := filepath.Join(work, "ws1", "tree")
|
||||
if err := os.MkdirAll(tree, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", "build")+"/.", tree).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
home := t.TempDir()
|
||||
m := &Module{Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
args := map[string]any{"workspace": "ws1", "module": ".", "artifact": "tools"}
|
||||
first, err := m.build(context.Background(), args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := m.build(context.Background(), args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, b := first.(BuildResult), second.(BuildResult)
|
||||
if a.Digest != b.Digest || a.Binary != "prog-bin" {
|
||||
t.Errorf("two builds of one tree: %+v and %+v", a, b)
|
||||
}
|
||||
raw, err := os.ReadFile(a.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tr := tar.NewReader(zr)
|
||||
h, err := tr.Next()
|
||||
if err != nil || h.Name != "prog-bin" || h.Mode&0o111 == 0 {
|
||||
t.Fatalf("first entry = %+v, %v", h, err)
|
||||
}
|
||||
if _, err := tr.Next(); err != io.EOF {
|
||||
t.Errorf("the archive holds more than one file")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// capturedRunner answers the commands of a test step from a fixture repository: ls-files with the go.mod, and every
|
||||
// `go test` with the output captured from the real go (testdata/captured, see its README).
|
||||
func capturedRunner(t *testing.T, captured string, calls *[]cmdRecord) forgekey.Runner {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "captured", captured+".json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status := 0
|
||||
if strings.Contains(string(raw), `"Action":"fail"`) {
|
||||
status = 1 // go test exits 1 when a test or a build failed
|
||||
}
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
*calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")})
|
||||
switch {
|
||||
case name == "git" && args[2] == "ls-files":
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
case name == "go":
|
||||
return forgekey.Ran{Stdout: string(raw), Status: status}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
}
|
||||
|
||||
// workspaceFor is a module whose work directory holds a fetched workspace, and the id of it.
|
||||
func workspaceFor(t *testing.T, captured string) (*Module, *keptStates, string, *[]cmdRecord) {
|
||||
t.Helper()
|
||||
calls := &[]cmdRecord{}
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
|
||||
m.Run = capturedRunner(t, captured, calls)
|
||||
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, states, "ws1", calls
|
||||
}
|
||||
|
||||
func runUnit(t *testing.T, m *Module, states *keptStates, ws string) (TestResult, delivery.Step) {
|
||||
t.Helper()
|
||||
var run func(map[string]any) (any, error)
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == "build_go_test" {
|
||||
run = tool.Run
|
||||
}
|
||||
}
|
||||
if _, err := run(map[string]any{"workspace": ws, "level": "unit"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res TestResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatalf("result %s: %v", st.Result, err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
// 1290/AC1: a fixture with one passing package and one failing test: the result names the failing test and its
|
||||
// package, and the level fails. The step itself finished, so it ends passed with the verdict in its result.
|
||||
func TestUnitLevelNamesTheFailingTestAndFailsTheLevel(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "failing")
|
||||
res, st := runUnit(t, m, states, ws)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if res.Passed || res.Level != "unit" {
|
||||
t.Errorf("result = %+v, want the unit level failed", res)
|
||||
}
|
||||
if len(res.Failed) != 1 || res.Failed[0].Package != "fixture/failing/bad" || res.Failed[0].Test != "TestThreeIsFour" ||
|
||||
!strings.Contains(res.Failed[0].Output, "Three() = 3, want 4") {
|
||||
t.Errorf("failed = %+v", res.Failed)
|
||||
}
|
||||
if !strings.Contains(res.Why, "fixture/failing/bad TestThreeIsFour") {
|
||||
t.Errorf("why = %q", res.Why)
|
||||
}
|
||||
good := false
|
||||
for _, p := range res.Packages {
|
||||
if p.Package == "fixture/failing/good" && p.Passed && p.Tests == 1 {
|
||||
good = true
|
||||
}
|
||||
}
|
||||
if !good || res.Tests != 2 {
|
||||
t.Errorf("packages = %+v, tests = %d; want the good package passed with its test, 2 tests", res.Packages, res.Tests)
|
||||
}
|
||||
}
|
||||
|
||||
// 1290/AC2: a fixture whose tests take more than the bound (lowered by the module's option, not by waiting five
|
||||
// seconds): the level fails naming the bound and the slowest tests; under the real bound the same run passes.
|
||||
func TestUnitLevelOverItsBoundFailsNamingTheBoundAndTheSlowestTests(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "slow")
|
||||
m.UnitBound = 10 * time.Millisecond
|
||||
res, _ := runUnit(t, m, states, ws)
|
||||
if !res.Passed || !res.OverBound || res.OverWhich != "run" {
|
||||
t.Fatalf("a level of %.3f s under a bound of %v: passed %v, over_bound %v (%s); the tests passed and the bound was kept no longer", res.Elapsed, m.UnitBound, res.Passed, res.OverBound, res.OverWhich)
|
||||
}
|
||||
for _, want := range []string{"over its bound of 0.01 s", "fixture/slow TestTakesAWhile"} {
|
||||
if !strings.Contains(res.Why, want) {
|
||||
t.Errorf("why = %q lacks %q", res.Why, want)
|
||||
}
|
||||
}
|
||||
if len(res.Slowest) == 0 || res.Slowest[0].Test != "TestTakesAWhile" || res.Bound != 0.01 {
|
||||
t.Errorf("slowest = %+v, bound = %v", res.Slowest, res.Bound)
|
||||
}
|
||||
|
||||
m2, states2, ws2, _ := workspaceFor(t, "slow")
|
||||
if res, _ := runUnit(t, m2, states2, ws2); !res.Passed || res.OverBound || res.Bound != 5 {
|
||||
t.Errorf("under the bound of five seconds: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// 1290/AC3: a fixture whose only test carries //go:build integration: go test says [no test files], and the unit
|
||||
// level reports no tests and passes. No command of the unit level asks for the tag.
|
||||
func TestUnitLevelOfAFixtureWithOnlyAnIntegrationTestHasNoTests(t *testing.T) {
|
||||
m, states, ws, calls := workspaceFor(t, "tagged")
|
||||
res, _ := runUnit(t, m, states, ws)
|
||||
if !res.Passed || !res.NoTests || res.Tests != 0 || !strings.Contains(res.Why, "no tests") {
|
||||
t.Errorf("result = %+v, want no tests, passed", res)
|
||||
}
|
||||
for _, c := range *calls {
|
||||
if strings.Contains(c.line, "-tags") {
|
||||
t.Errorf("the unit level asked for a tag: %s", c.line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1290/AC4: a fixture with no tests at all: the result says the repository has no tests, neither a failure nor a warning.
|
||||
func TestUnitLevelOfAFixtureWithNoTestsSaysSo(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "empty")
|
||||
res, st := runUnit(t, m, states, ws)
|
||||
if st.State != delivery.StepPassed || !res.Passed || !res.NoTests || res.Why != "the repository has no tests at the unit level" || len(res.Failed) != 0 {
|
||||
t.Errorf("result = %+v (%s)", res, st.State)
|
||||
}
|
||||
}
|
||||
|
||||
// A package that does not build fails the level with the compiler's words.
|
||||
func TestUnitLevelOfAPackageThatDoesNotBuildFailsWithTheCompilersLine(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "broken")
|
||||
res, _ := runUnit(t, m, states, ws)
|
||||
if res.Passed || len(res.Failed) != 1 || res.Failed[0].Package != "fixture/broken" ||
|
||||
!strings.Contains(res.Failed[0].Output, `cannot use "one" (untyped string constant) as int value`) {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// The command is go test -json in the module's directory of the workspace, with the toolchain pinned to the installed
|
||||
// one and the module's caches.
|
||||
func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *testing.T) {
|
||||
m, states, ws, calls := workspaceFor(t, "empty")
|
||||
runUnit(t, m, states, ws)
|
||||
var line string
|
||||
var env string
|
||||
for _, c := range *calls {
|
||||
if strings.HasPrefix(c.line, "go ") {
|
||||
line, env = c.line, strings.Join(c.env, " ")
|
||||
}
|
||||
}
|
||||
want := "go -C " + filepath.Join(m.Work, ws, "tree", ".") + " test -json -count=1 -timeout 10m ./..."
|
||||
if line != want {
|
||||
t.Errorf("command = %q, want %q", line, want)
|
||||
}
|
||||
for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod")} {
|
||||
if !strings.Contains(env, w) {
|
||||
t.Errorf("environment %s lacks %s", env, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A go test that said nothing a test says (it could not run) ends the step failed, not the level.
|
||||
func TestAGoTestThatCouldNotRunEndsTheStepFailed(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "empty")
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name == "go" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "go: cannot find main module\n"}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
}
|
||||
_, st := runUnit(t, m, states, ws)
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "go: cannot find main module") {
|
||||
t.Errorf("the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATestOfAnUnknownWorkspaceOrAnotherLevelIsRefused(t *testing.T) {
|
||||
m, states, _, _ := workspaceFor(t, "empty")
|
||||
var run func(map[string]any) (any, error)
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == "build_go_test" {
|
||||
run = tool.Run
|
||||
}
|
||||
}
|
||||
for _, args := range []map[string]any{
|
||||
{"workspace": "nope", "level": "unit"},
|
||||
{"workspace": "../ws1", "level": "unit"},
|
||||
{"workspace": "ws1", "level": "stress"},
|
||||
{"workspace": "ws1", "level": "unit", "packages": []any{"-run=x"}},
|
||||
{"workspace": "ws1", "level": "unit", "packages": []any{"../x"}},
|
||||
} {
|
||||
run(args)
|
||||
if st := states.waitEnd(t); st.State != delivery.StepFailed {
|
||||
t.Errorf("%v: the step ended %s", args, st.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A directory that vendors is tested, and listed, from its vendor directory: fetch downloaded nothing for it, and the
|
||||
// last value of GOFLAGS is the one go uses.
|
||||
func TestADirectoryThatVendorsIsReadFromItsVendorDirectory(t *testing.T) {
|
||||
m, states, ws, calls := workspaceFor(t, "empty")
|
||||
vendor := filepath.Join(m.Work, ws, "tree", "vendor")
|
||||
os.MkdirAll(vendor, 0o755)
|
||||
os.WriteFile(filepath.Join(vendor, "modules.txt"), nil, 0o644)
|
||||
runUnit(t, m, states, ws)
|
||||
flags := ""
|
||||
for _, c := range *calls {
|
||||
if strings.HasPrefix(c.line, "go ") {
|
||||
for _, w := range c.env {
|
||||
if strings.HasPrefix(w, "GOFLAGS=") {
|
||||
flags = w
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if flags != "GOFLAGS=-mod=vendor" {
|
||||
t.Errorf("the last GOFLAGS of go test in a vendoring directory is %q", flags)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
Each file is the standard output of `GOTOOLCHAIN=local GOFLAGS=-mod=mod go test -count=1 -json ./...` (go1.27.1)
|
||||
run in the fixture module of the same name under ../repos, captured on 2026-10-20 for issue 1290. The unit
|
||||
tests feed them to the parser through a fake runner, so no test here runs `go test`.
|
||||
|
||||
golist-external.json is `GOTOOLCHAIN=local GOFLAGS=-mod=mod go list -json=ImportPath,Standard,DepOnly,Module
|
||||
git.novox.be/novox/mesh-sdk/go/buildstep` run in this module (go1.27.1, 2026-10-20 for issue 1291): a package of a module
|
||||
outside the tree as go list describes it.
|
||||
|
||||
toolchain-go-entry.txt is lines 155-191 of mesh-control's internal/builder/toolchain.go at origin/main 073ccc29 (fetched
|
||||
2026-10-20): the Go entry of today's build agent's recipe, which the build step's flags are compared with.
|
||||
@@ -0,0 +1,6 @@
|
||||
{"ImportPath":"fixture/broken [fixture/broken.test]","Action":"build-output","Output":"# fixture/broken [fixture/broken.test]\n"}
|
||||
{"ImportPath":"fixture/broken [fixture/broken.test]","Action":"build-output","Output":"./broken.go:3:25: cannot use \"one\" (untyped string constant) as int value in return statement\n"}
|
||||
{"ImportPath":"fixture/broken [fixture/broken.test]","Action":"build-fail"}
|
||||
{"Time":"2026-10-20T22:20:06.344233803+02:00","Action":"start","Package":"fixture/broken"}
|
||||
{"Time":"2026-10-20T22:20:06.344344815+02:00","Action":"output","Package":"fixture/broken","Output":"FAIL\tfixture/broken [build failed]\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.344358865+02:00","Action":"fail","Package":"fixture/broken","Elapsed":0,"FailedBuild":"fixture/broken [fixture/broken.test]"}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"Time":"2026-10-20T22:20:06.064193534+02:00","Action":"start","Package":"fixture/empty"}
|
||||
{"Time":"2026-10-20T22:20:06.064489419+02:00","Action":"output","Package":"fixture/empty","Output":"? \tfixture/empty\t[no test files]\n"}
|
||||
{"Time":"2026-10-20T22:20:06.064510249+02:00","Action":"skip","Package":"fixture/empty","Elapsed":0}
|
||||
@@ -0,0 +1,17 @@
|
||||
{"Time":"2026-10-20T22:20:06.028267402+02:00","Action":"start","Package":"fixture/failing/bad"}
|
||||
{"Time":"2026-10-20T22:20:06.028528997+02:00","Action":"start","Package":"fixture/failing/good"}
|
||||
{"Time":"2026-10-20T22:20:06.030412986+02:00","Action":"run","Package":"fixture/failing/bad","Test":"TestThreeIsFour"}
|
||||
{"Time":"2026-10-20T22:20:06.030453876+02:00","Action":"output","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Output":"=== RUN TestThreeIsFour\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.030494057+02:00","Action":"output","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Output":" bad_test.go:7: Three() = 3, want 4\n","OutputType":"error"}
|
||||
{"Time":"2026-10-20T22:20:06.030511077+02:00","Action":"output","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Output":"--- FAIL: TestThreeIsFour (0.00s)\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.030519017+02:00","Action":"fail","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Elapsed":0}
|
||||
{"Time":"2026-10-20T22:20:06.030529158+02:00","Action":"output","Package":"fixture/failing/bad","Output":"FAIL\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.030546038+02:00","Action":"run","Package":"fixture/failing/good","Test":"TestTwo"}
|
||||
{"Time":"2026-10-20T22:20:06.030561528+02:00","Action":"output","Package":"fixture/failing/good","Test":"TestTwo","Output":"=== RUN TestTwo\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.030570028+02:00","Action":"output","Package":"fixture/failing/good","Test":"TestTwo","Output":"--- PASS: TestTwo (0.00s)\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.030577128+02:00","Action":"pass","Package":"fixture/failing/good","Test":"TestTwo","Elapsed":0}
|
||||
{"Time":"2026-10-20T22:20:06.030583418+02:00","Action":"output","Package":"fixture/failing/good","Output":"PASS\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.030962814+02:00","Action":"output","Package":"fixture/failing/bad","Output":"FAIL\tfixture/failing/bad\t0.002s\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.030974515+02:00","Action":"output","Package":"fixture/failing/good","Output":"ok \tfixture/failing/good\t0.002s\n"}
|
||||
{"Time":"2026-10-20T22:20:06.030991305+02:00","Action":"pass","Package":"fixture/failing/good","Elapsed":0.002}
|
||||
{"Time":"2026-10-20T22:20:06.030982975+02:00","Action":"fail","Package":"fixture/failing/bad","Elapsed":0.003}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"ImportPath": "git.novox.be/novox/mesh-sdk/go/buildstep",
|
||||
"Module": {
|
||||
"Path": "git.novox.be/novox/mesh-sdk/go",
|
||||
"Version": "v0.1.14-0.20261020200715-9e296295c05e",
|
||||
"Time": "2026-10-20T20:07:15Z",
|
||||
"Dir": "/home/jochen/go/pkg/mod/git.novox.be/novox/mesh-sdk/go@v0.1.14-0.20261020200715-9e296295c05e",
|
||||
"GoMod": "/home/jochen/go/pkg/mod/cache/download/git.novox.be/novox/mesh-sdk/go/@v/v0.1.14-0.20261020200715-9e296295c05e.mod",
|
||||
"GoVersion": "1.26.0",
|
||||
"Sum": "h1:uf2o2A6dBOrISDUVw6U7+fv8ylwwF+UfRrGWKIQMFOE=",
|
||||
"GoModSum": "h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY="
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{"Time":"2026-10-20T22:20:06.21802535+02:00","Action":"start","Package":"fixture/slow"}
|
||||
{"Time":"2026-10-20T22:20:06.222338257+02:00","Action":"run","Package":"fixture/slow","Test":"TestTakesAWhile"}
|
||||
{"Time":"2026-10-20T22:20:06.222387677+02:00","Action":"output","Package":"fixture/slow","Test":"TestTakesAWhile","Output":"=== RUN TestTakesAWhile\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.282860204+02:00","Action":"output","Package":"fixture/slow","Test":"TestTakesAWhile","Output":"--- PASS: TestTakesAWhile (0.06s)\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.282917774+02:00","Action":"pass","Package":"fixture/slow","Test":"TestTakesAWhile","Elapsed":0.06}
|
||||
{"Time":"2026-10-20T22:20:06.282937215+02:00","Action":"run","Package":"fixture/slow","Test":"TestQuick"}
|
||||
{"Time":"2026-10-20T22:20:06.282943535+02:00","Action":"output","Package":"fixture/slow","Test":"TestQuick","Output":"=== RUN TestQuick\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.282955095+02:00","Action":"output","Package":"fixture/slow","Test":"TestQuick","Output":"--- PASS: TestQuick (0.00s)\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.282963365+02:00","Action":"pass","Package":"fixture/slow","Test":"TestQuick","Elapsed":0}
|
||||
{"Time":"2026-10-20T22:20:06.282971185+02:00","Action":"output","Package":"fixture/slow","Output":"PASS\n","OutputType":"frame"}
|
||||
{"Time":"2026-10-20T22:20:06.283463443+02:00","Action":"output","Package":"fixture/slow","Output":"ok \tfixture/slow\t0.063s\n"}
|
||||
{"Time":"2026-10-20T22:20:06.283481213+02:00","Action":"pass","Package":"fixture/slow","Elapsed":0.065}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"Time":"2026-10-20T22:20:06.049418313+02:00","Action":"start","Package":"fixture/tagged"}
|
||||
{"Time":"2026-10-20T22:20:06.049698258+02:00","Action":"output","Package":"fixture/tagged","Output":"? \tfixture/tagged\t[no test files]\n"}
|
||||
{"Time":"2026-10-20T22:20:06.049730188+02:00","Action":"skip","Package":"fixture/tagged","Elapsed":0}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
Language: "go",
|
||||
Base: "mesh-tools-go",
|
||||
Artifact: "build",
|
||||
// **The mesh's own components, and not modules.** The warning above this list — that every
|
||||
// language is another implementation of the contracts modules share, so adding one commits
|
||||
// to keeping N implementations in step — does not attach here. Go is how the host, the
|
||||
// control plane and the builder are written, and none of them is a module in that sense:
|
||||
// the host is what APPLIES modules. So there is no SDK obligation, and the reason this
|
||||
// entry did not exist was that nothing needed to compile the mesh itself
|
||||
// (novox/hq ADR 0142, and 04-ISSUES/142 where that is why nothing delivers the host).
|
||||
//
|
||||
// Static, because what a machine ends up holding is a file rather than a container, and a
|
||||
// binary that needs a libc it did not bring is a delivery that works until a machine
|
||||
// differs. Trimmed of its own paths for the same reason a version comes from where it sits
|
||||
// rather than from the linker: two builds of one commit produce the same bytes.
|
||||
//
|
||||
// **And of one build source at two commits** (novox/hq ADR 0267 rule 6, issue 363): a module built
|
||||
// from its repository's root is compiled with the clone's `.git` in view, and Go stamps the commit,
|
||||
// its time and whether the tree was modified into the binary unless told not to — so a merge that
|
||||
// changed only a README made a new digest of an unchanged program.
|
||||
Compile: []string{
|
||||
"env", "CGO_ENABLED=0", "GOFLAGS=-trimpath -buildvcs=false",
|
||||
"go", "build",
|
||||
},
|
||||
// Stripped of symbols and debug info: what a machine holds is a file it runs, not one it
|
||||
// debugs, and the difference measured 12.2MB against 8.5MB.
|
||||
LinkerFlags: []string{"-s", "-w"},
|
||||
OutputFlag: "-o",
|
||||
// Pointed at the package the artifact is built `from`, compiled whole. Go writes the binary
|
||||
// into the output directory, named after the package — so the bundle a machine unpacks is a
|
||||
// directory holding one executable, which is what the delivery mechanism expects
|
||||
// (novox/hq ADR 0141).
|
||||
Unit: UnitPackage,
|
||||
// The mesh's own Go components read the system they were built for from this variable, and
|
||||
// refuse to touch a machine without one.
|
||||
SystemStamp: "main.builtFor",
|
||||
},
|
||||
@@ -0,0 +1,3 @@
|
||||
package broken
|
||||
|
||||
func One() int { return "one" }
|
||||
@@ -0,0 +1,5 @@
|
||||
package broken
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestOne(t *testing.T) { One() }
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/broken
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,9 @@
|
||||
// Command prog is what the build step of issue 1487 builds in its integration test.
|
||||
package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
// builtFor is stamped by the linker.
|
||||
var builtFor string
|
||||
|
||||
func main() { fmt.Println("prog built for", builtFor) }
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/build
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"module": "fixture-build",
|
||||
"version": "1",
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/prog",
|
||||
"binary": "prog-bin",
|
||||
"loads": ["prog-bin"]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
package a
|
||||
|
||||
// A is what a's tests check.
|
||||
func A() int { return 1 }
|
||||
@@ -0,0 +1,15 @@
|
||||
package a
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"fixture/digest/b"
|
||||
)
|
||||
|
||||
func TestAReadsBAndItsTestdata(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/x.txt")
|
||||
if err != nil || A()+b.B() != 3 || len(raw) == 0 {
|
||||
t.Fatal("a")
|
||||
}
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
x
|
||||
@@ -0,0 +1,4 @@
|
||||
package b
|
||||
|
||||
// B is reached by a's tests.
|
||||
func B() int { return 2 }
|
||||
@@ -0,0 +1,6 @@
|
||||
package c
|
||||
|
||||
import _ "embed"
|
||||
|
||||
//go:embed note.txt
|
||||
var Note string
|
||||
@@ -0,0 +1,9 @@
|
||||
package c
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCHasANote(t *testing.T) {
|
||||
if Note == "" {
|
||||
t.Fatal("c")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
note
|
||||
@@ -0,0 +1,4 @@
|
||||
package d
|
||||
|
||||
// D is reached by no test.
|
||||
func D() int { return 4 }
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/digest
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,3 @@
|
||||
package empty
|
||||
|
||||
func Nothing() {}
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/empty
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,3 @@
|
||||
package bad
|
||||
|
||||
func Three() int { return 3 }
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package bad
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestThreeIsFour(t *testing.T) {
|
||||
if Three() != 4 {
|
||||
t.Fatalf("Three() = %d, want 4", Three())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/failing
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,3 @@
|
||||
package good
|
||||
|
||||
func Two() int { return 2 }
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package good
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTwo(t *testing.T) {
|
||||
if Two() != 2 {
|
||||
t.Fatal("two")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/slow
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,3 @@
|
||||
package slow
|
||||
|
||||
func Zero() int { return 0 }
|
||||
@@ -0,0 +1,16 @@
|
||||
package slow
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTakesAWhile(t *testing.T) {
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
}
|
||||
|
||||
func TestQuick(t *testing.T) {
|
||||
if Zero() != 0 {
|
||||
t.Fatal("zero")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/store
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,4 @@
|
||||
package store
|
||||
|
||||
// Name is what the integration test below finds a store and a bus for.
|
||||
const Name = "store"
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
//go:build integration
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The fixture of issue 1486: the store and the bus a run gives its tests are there, reached as a test reaches them.
|
||||
func TestTheStoreAndTheBusAreThere(t *testing.T) {
|
||||
u, err := url.Parse(os.Getenv("MESH_TEST_POSTGRES"))
|
||||
if err != nil || u.Query().Get("host") == "" {
|
||||
t.Fatalf("MESH_TEST_POSTGRES = %q", os.Getenv("MESH_TEST_POSTGRES"))
|
||||
}
|
||||
pg, err := net.DialTimeout("unix", filepath.Join(u.Query().Get("host"), ".s.PGSQL.5432"), 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("the store's socket: %v", err)
|
||||
}
|
||||
pg.Close()
|
||||
|
||||
bus, err := net.DialTimeout("tcp", strings.TrimPrefix(os.Getenv("MESH_TEST_NATS"), "nats://"), 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("the bus: %v", err)
|
||||
}
|
||||
defer bus.Close()
|
||||
line, err := bufio.NewReader(bus).ReadString('\n')
|
||||
if err != nil || !strings.HasPrefix(line, "INFO ") || !strings.Contains(line, `"jetstream":true`) {
|
||||
t.Fatalf("the bus said %q (%v), want an INFO line with JetStream on", line, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/tagged
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,3 @@
|
||||
package tagged
|
||||
|
||||
func One() int { return 1 }
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
//go:build integration
|
||||
|
||||
package tagged
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNeedsAStore(t *testing.T) {
|
||||
if One() != 1 {
|
||||
t.Fatal("one")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// The tools this module serves. A tool is named as the module is: build_go_ and its verb (a tool name has no dot).
|
||||
const (
|
||||
StatusTool = "build_go_status"
|
||||
HealthTool = "build_go_health"
|
||||
)
|
||||
|
||||
// StepTool is one step tool of the module: fetch, test, digest or build. Long is a step that answers at once and
|
||||
// ends on its key (ADR 0469 §2); the others answer in their call. Both run behind the toolchain guard.
|
||||
type StepTool struct {
|
||||
Name string
|
||||
Description string
|
||||
Input map[string]any
|
||||
Long bool
|
||||
Run func(ctx context.Context, args map[string]any) (any, error)
|
||||
}
|
||||
|
||||
// StepTools are the module's step tools. Each step's issue adds its own.
|
||||
func (m *Module) StepTools() []StepTool {
|
||||
return []StepTool{m.fetchTool(), m.testTool(), m.digestTool(), m.buildTool()}
|
||||
}
|
||||
|
||||
// Tools is every tool the module serves: its step tools behind the guard, its status and its health.
|
||||
func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools := []stdio.Tool{}
|
||||
for _, s := range steps {
|
||||
t := stdio.Tool{Name: s.Name, Description: s.Description, Input: s.Input}
|
||||
if s.Long {
|
||||
t.Run = m.Long(s.Name, s.Run)
|
||||
} else {
|
||||
t.Run = m.Short(s.Run)
|
||||
}
|
||||
tools = append(tools, t)
|
||||
}
|
||||
return append(tools,
|
||||
stdio.Tool{Name: StatusTool,
|
||||
Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " +
|
||||
"the recipe version, the postgres major version and the nats-server version the integration level uses, and " +
|
||||
"why it refuses to build if it does. Read-only.",
|
||||
Run: func(map[string]any) (any, error) { return m.Status(m.Ctx), nil }},
|
||||
stdio.Tool{Name: HealthTool,
|
||||
Description: "How the Go build module is: unhealthy while the Go installed on this machine is not the version its " +
|
||||
"recipe builds with, saying both versions; healthy otherwise. Read-only.",
|
||||
Run: func(map[string]any) (any, error) { return m.Health(m.Ctx), nil }},
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
Host ${bound:forge-account:ssh_alias}
|
||||
HostName ${bound:forge-account:ssh_host}
|
||||
Port ${bound:forge-account:ssh_port}
|
||||
HostKeyAlias ${bound:forge-account:ssh_alias}
|
||||
User git
|
||||
IdentityFile ~/.ssh/mesh_ed25519
|
||||
IdentitiesOnly yes
|
||||
@@ -0,0 +1,2 @@
|
||||
[url "ssh://${bound:forge-account:ssh_alias}/"]
|
||||
insteadOf = ${bound:forge-account:https_base}
|
||||
@@ -0,0 +1 @@
|
||||
${bound:forge-account:ssh_alias} ${bound:forge-account:host_key}
|
||||
@@ -0,0 +1,21 @@
|
||||
module meshbuildgo
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020200715-9e296295c05e h1:uf2o2A6dBOrISDUVw6U7+fv8ylwwF+UfRrGWKIQMFOE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020200715-9e296295c05e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020203945-1572e50e0ef0 h1:mr9HyARwzeZT7FxFHv1Xc2z7QErvDZ/F/pBMIpjGSd0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020203945-1572e50e0ef0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d h1:mmOCJphjQN15jNVh+sMQqnmJ3UiYltiuQRg/zw3wFtw=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,148 @@
|
||||
{
|
||||
"module": "mesh-build-go",
|
||||
"version": "1",
|
||||
"runs-as": "mesh-build-go",
|
||||
"capabilities": [
|
||||
"package-manager",
|
||||
"user-namespaces"
|
||||
],
|
||||
"state": [
|
||||
"toolchains",
|
||||
"keys",
|
||||
"steps"
|
||||
],
|
||||
"tools": [
|
||||
"build_go_fetch",
|
||||
"build_go_test",
|
||||
"build_go_digest",
|
||||
"build_go_build",
|
||||
"build_go_status",
|
||||
"build_go_health"
|
||||
],
|
||||
"streams": [
|
||||
{
|
||||
"name": "toolchains-go",
|
||||
"events": ["toolchain.**"]
|
||||
}
|
||||
],
|
||||
"requires": [
|
||||
"forge-account"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-build-go",
|
||||
"runtime-health": {
|
||||
"kind": "tool",
|
||||
"tool": "build_go_health",
|
||||
"interval": "60s",
|
||||
"timeout": "10s",
|
||||
"looks": 2,
|
||||
"grace": "60s"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-build-go",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-build-go"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-build-go",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "work",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-build-go-work",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-build-go",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "cache",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-build-go-cache",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-build-go",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "ssh-dir",
|
||||
"type": "directory",
|
||||
"path": "${dir:state}/.ssh",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-build-go",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "ssh-config",
|
||||
"type": "file",
|
||||
"path": "${dir:ssh-dir}/config",
|
||||
"owner": "mesh-build-go",
|
||||
"mode": "0600",
|
||||
"template": "files/forge"
|
||||
},
|
||||
{
|
||||
"id": "known-hosts",
|
||||
"type": "file",
|
||||
"path": "${dir:ssh-dir}/known_hosts",
|
||||
"owner": "mesh-build-go",
|
||||
"mode": "0644",
|
||||
"template": "files/known_hosts"
|
||||
},
|
||||
{
|
||||
"id": "git-config",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/.config/git/config",
|
||||
"owner": "mesh-build-go",
|
||||
"mode": "0644",
|
||||
"template": "files/gitconfig"
|
||||
},
|
||||
{
|
||||
"id": "go",
|
||||
"type": "package",
|
||||
"package": "go"
|
||||
},
|
||||
{
|
||||
"id": "git",
|
||||
"type": "package",
|
||||
"package": "git"
|
||||
},
|
||||
{
|
||||
"id": "openssh",
|
||||
"type": "package",
|
||||
"package": "openssh"
|
||||
},
|
||||
{
|
||||
"id": "postgresql",
|
||||
"type": "package",
|
||||
"package": "postgresql"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-build-go",
|
||||
"binary": "mesh-build-go",
|
||||
"loads": [
|
||||
"mesh-build-go"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
# mesh-build-node — the Node and TypeScript build module
|
||||
|
||||
Design 47, release R4c-3 (novox/hq ADR 0405, ADR 0466 §1, ADR 0469 §1 and §4). The module that builds and checks
|
||||
TypeScript code on a build machine, as an account of its own (`runs-as`, no login shell, home at mode 0700), with the
|
||||
packages it declares (`nodejs`, `npm`, `git`, `openssh`) and the capability `user-namespaces`. The bundle is Go
|
||||
(`cmd/mesh-build-node`). It claims no seat: it is assigned to the build machines (ADR 0469 §6) and runs dark until the
|
||||
builder uses it. Its forge key and ssh files are the same as mesh-build-go's (issue 1484).
|
||||
|
||||
## The recipe
|
||||
|
||||
`recipe.go` names the one Node version it builds with (`RecipeNode`, `v26.10.0`, as `node --version` prints it) and the
|
||||
recipe's own version (`RecipeSteps`). The recipe version is a hash of those two and of the bytes of the lock file
|
||||
`toolchain/package-lock.json`, so one changed byte of the lock, another Node or another recipe moves it.
|
||||
|
||||
## The toolchain directory
|
||||
|
||||
`toolchain/package.json` and its committed `package-lock.json` name typescript 5.9.3, esbuild 0.25.12, @types/node 22.20.1 (the types tsc checks node imports against),
|
||||
@novox/mesh-sdk 0.1.14 and nats 2.29.3. The mesh unpacks both, as the archive artifact `toolchain`, into `<home>/toolchain/` (resource `toolchain-dir`; a
|
||||
manifest names no host, and the lock does, so it is not a template); the program reads the lock from there. When `<cache dir>/toolchain/<recipe version>` is missing
|
||||
the module runs `npm ci --ignore-scripts` there once (a directory without its `.installed` mark counts as missing).
|
||||
|
||||
**How the registry is reached** (the choice of issue 1488): the lock holds @novox/mesh-sdk at the forge's public
|
||||
address, and npm 12 installs a locked tarball only from the registry it is configured with. The module therefore
|
||||
requires `npm-package-registry` (read access, provided by gitea) and, like the lab module, takes its binding and sealed
|
||||
secret as files in its home. The install gets a private npm config file with the `@novox` scope at the binding's
|
||||
`public-root` (else scheme, address, port and npm path) and the bound account's credential, and removes it afterwards;
|
||||
every other package resolves from the public registry. This is the way today's build agent names it.
|
||||
|
||||
The runtime tree for bundles with externals (the pruned sdk and nats today in `/app/runtime`) is not installed
|
||||
separately: sdk and nats are in this directory's `node_modules`, and the build step (issue 1490) takes them from there.
|
||||
|
||||
## The guard and the health
|
||||
|
||||
At start, every ten minutes, before every step and at every health look, the module reads the lock again and the
|
||||
installed Node (`node --version`) and writes it with the recipe version to `<node>.typescript` of its `toolchains`
|
||||
state. A machine whose Node is not the recipe's refuses every step, the step ending `refused` on its key in `steps` with
|
||||
both versions named, and `build_node_health` (the module's `runtime-health`) answers unhealthy with the same text, so
|
||||
the controller raises the machine's condition. A step runs only after the toolchain directory is installed.
|
||||
|
||||
## Tools
|
||||
|
||||
`build_node_status` and `build_node_health` answer in their call. The step tools run behind the guard and the
|
||||
toolchain install: a long step answers at once with its id and ends on `<node>.<id>` of `steps` (ADR 0469 §2).
|
||||
|
||||
- `build_node_fetch` {repository, commit, ref} (issue 1489): the commit over ssh as the module's account
|
||||
(`buildstep.FetchCommit`, remote `ssh://mesh-forge/<owner>/<name>.git`) into `<work>/<id>/tree`, then for each
|
||||
TypeScript module (a module.json with a typescript artifact) the packages its package.json names beyond the SDK,
|
||||
`typescript` and `@types/*` (the toolchain carries those): none installs nothing; some need a package-lock.json, and
|
||||
are installed as the build agent does (scratch copy without the SDK,
|
||||
devDependencies and scripts; `npm ci --omit=dev --omit=peer --ignore-scripts --no-bin-links`; copies of the SDK
|
||||
removed). The mesh's registry reaches `npm ci` through a private config file (`--userconfig`), so
|
||||
other `@novox/` packages install. A module with packages and no lock does not fail the fetch: its result lists it
|
||||
under `refused` with the reason, and a build of that bundle fails naming it (issue 1459). Every command runs through `forgekey.ExecRunnerClean(0)` with PATH, HOME, the locale and its own
|
||||
words, nothing of this process's environment. Workspaces older than a day are removed at each fetch.
|
||||
- `build_node_test` {workspace, module, level} (issue 1491): the unit level is the type-check (the toolchain's tsc, the
|
||||
package's `tsconfig.json` or the build agent's flags over its .ts files) and the `test` script; the integration level
|
||||
is the `test:integration` script, the type-check being its setup. Bounds (the Go module's; ADR 0445 names none for
|
||||
TypeScript): unit 5 s, integration 30 s setup and 60 s running. The test script finds `tsc` and `esbuild` on its PATH through `<work>/<id>/bin` (shell scripts calling the toolchain's). A level over its bound says `over_bound` and passes;
|
||||
a failed one finishes the step with `passed` false and the output (the step fails only when the tests could not be run). Neither script nor .ts files: no tests. The toolchain makes no bin
|
||||
links, so a script runs `node` and files, not `tsc` by name.
|
||||
- `build_node_build` {workspace, module, artifact} (issue 1490): the build agent's tsc and esbuild lines
|
||||
(mesh-controller internal/builder, cited in `build.go`), the launchers, `package.json`, `buildstep.Pack`, and the
|
||||
digest. No network. The build runs in a staging copy of the module at `<work>/<id>/stage-<artifact>/modules/module`, with the
|
||||
toolchain's packages (not typescript or esbuild) copied to `stage-<artifact>/node_modules`: the build agent's depth
|
||||
(`/app/modules/module`, `/app/node_modules`), so esbuild's `../../node_modules/...` path comments, and the bundle's
|
||||
digest, equal its. Copies, not links, because the mesh makes no symlink. A bundle that keeps an external gets them in its own `node_modules`.
|
||||
The packed archive is written beside the tree as `<artifact>.tar.gz`.
|
||||
@@ -0,0 +1,283 @@
|
||||
package main
|
||||
|
||||
// The build step (novox/hq ADR 0405 §3 and §7, ADR 0469 §4; issue 1490): a TypeScript bundle of a fetched workspace
|
||||
// compiled with the toolchain's tsc, bundled with its esbuild, given its launchers, packed with mesh-sdk's Pack and
|
||||
// ended with the archive's digest. No network. The commands and flags are the build agent's, copied from
|
||||
// mesh-controller origin/main internal/builder: toolchain.go:143-147 (tsc), builder.go:1637-1705 (compile),
|
||||
// :1926-1956 (writeLaunchers), :1971-2039 (bundled), :1735-1747 (sourcesFor). Where they ran in the toolchain image
|
||||
// they run here from the module's directory with the toolchain's real paths, and tsc and esbuild find the SDK in the
|
||||
// workspace's node_modules (workspace.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// launcherSuffix is what an entrypoint's launcher is called beside it: index.js is started as index.serve.mjs
|
||||
// (novox/hq ADR 0193).
|
||||
const launcherSuffix = ".serve.mjs"
|
||||
|
||||
// Out is where a bundle's compiled files go in the module's directory (builder toolchain.go:118).
|
||||
func Out(artifact string) string { return ".mesh-build/" + artifact }
|
||||
|
||||
// banner is esbuild's createRequire shim: CommonJS inlined into an ES module still finds `require`.
|
||||
const banner = "--banner:js=import { createRequire as __meshRequire } from 'node:module'; const require = __meshRequire(import.meta.url);"
|
||||
|
||||
// artifact is the part of a module.json artifact the build reads.
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Language string `json:"language"`
|
||||
Entrypoints []string `json:"entrypoints"`
|
||||
External []string `json:"external"`
|
||||
}
|
||||
|
||||
// BuildResult is what a passed build step says.
|
||||
type BuildResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Module string `json:"module"`
|
||||
Artifact string `json:"artifact"`
|
||||
// Digest is "sha256:" and the hex of the archive's bytes (buildstep.Pack).
|
||||
Digest string `json:"digest"`
|
||||
Bytes int `json:"bytes"`
|
||||
// Archive is the packed file, in the workspace beside its tree.
|
||||
Archive string `json:"archive"`
|
||||
Files []string `json:"files"`
|
||||
}
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_node_build",
|
||||
Description: "Build a TypeScript bundle of a fetched workspace: compile it with the toolchain's tsc, bundle each " +
|
||||
"entrypoint with its esbuild, write the launchers and package.json, pack the bundle and answer its digest. " +
|
||||
"Answers a step id at once; the step ends on its key in the steps state with the digest. No network.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch step answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, '.' when it is the root"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the name of the bundle in the module's module.json"},
|
||||
},
|
||||
"required": []string{"workspace", "artifact"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.build,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
root, dir, err := m.moduleDirOf(workspace, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := readArtifact(dir, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why, err := lockProblem(dir, module); err != nil || why != "" {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("%s was refused by the fetch: %s", name, why)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
// Built at the build agent's depth (builder.go: the module at /app/modules/module, the packages at /app/node_modules),
|
||||
// so the path comments esbuild writes, `../../node_modules/...`, and with them the bundle's bytes equal its.
|
||||
stage := filepath.Join(root, "stage-"+a.Name)
|
||||
if err := os.RemoveAll(stage); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
src := dir
|
||||
dir = filepath.Join(stage, "modules", "module")
|
||||
if err := copyTree(src, dir); err != nil {
|
||||
return nil, fmt.Errorf("staging the module: %w", err)
|
||||
}
|
||||
if err := copyPackages(filepath.Join(m.toolchainDir(), "node_modules"), filepath.Join(stage, "node_modules")); err != nil {
|
||||
return nil, fmt.Errorf("providing the toolchain's packages to the build: %w", err)
|
||||
}
|
||||
out, final := Out(a.Name), Out(a.Name)+".bundled"
|
||||
if err := os.MkdirAll(filepath.Join(dir, out), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// tsc: builder.go:1637-1705 with toolchain.go:143-147's flags, rooted at the module so an entrypoint lands where
|
||||
// it is named.
|
||||
tsc := []string{m.tsc(), "--module", "NodeNext", "--moduleResolution", "NodeNext", "--target", "ES2022",
|
||||
"--rootDir", ".", "--outDir", out}
|
||||
tsc = append(tsc, sourcesFor(a.Entrypoints, out, ".ts")...)
|
||||
if why := ranFailed(m.runIn(ctx, dir, nil, "node", tsc...)); why != "" {
|
||||
return nil, fmt.Errorf("tsc for %s: %s", a.Name, why)
|
||||
}
|
||||
launchers, err := writeLaunchers(filepath.Join(dir, out), a)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.bundle(ctx, dir, a, launchers); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bundleDir := filepath.Join(dir, final)
|
||||
if len(a.External) > 0 {
|
||||
if err := copyPackages(filepath.Join(m.toolchainDir(), "node_modules"), filepath.Join(bundleDir, "node_modules")); err != nil {
|
||||
return nil, fmt.Errorf("copying the packages %s keeps external: %w", a.Name, err)
|
||||
}
|
||||
}
|
||||
archive, digest, err := buildstep.Pack(bundleDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("packing %s: %w", a.Name, err)
|
||||
}
|
||||
file := filepath.Join(root, a.Name+".tar.gz")
|
||||
if err := os.WriteFile(file, archive, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files := []string{}
|
||||
_ = filepath.WalkDir(bundleDir, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() {
|
||||
rel, _ := filepath.Rel(bundleDir, p)
|
||||
files = append(files, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(files)
|
||||
return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive), Archive: file, Files: files}, nil
|
||||
}
|
||||
|
||||
// bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is
|
||||
// bundled in place under its own name, a launcher keeps its name with the extension .mjs.
|
||||
func (m *Module) bundle(ctx context.Context, dir string, a artifact, launchers map[string]string) error {
|
||||
out, final := Out(a.Name), Out(a.Name)+".bundled"
|
||||
if err := os.RemoveAll(filepath.Join(dir, final)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, final), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
common := []string{"--bundle", "--platform=node", "--format=esm", "--target=node22",
|
||||
"--outbase=" + out, "--outdir=" + final, "--log-level=warning", banner}
|
||||
for _, x := range a.External {
|
||||
common = append(common, "--external:"+x)
|
||||
}
|
||||
var plain, launch []string
|
||||
for _, e := range a.Entrypoints {
|
||||
if strings.HasSuffix(e, ".js") {
|
||||
plain = append(plain, out+"/"+e)
|
||||
}
|
||||
}
|
||||
for _, l := range sortedValues(launchers) {
|
||||
launch = append(launch, out+"/"+l)
|
||||
}
|
||||
step := func(entries []string, extra ...string) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
args := append(append(append([]string{}, entries...), common...), extra...)
|
||||
if why := ranFailed(m.runIn(ctx, dir, nil, m.esbuild(), args...)); why != "" {
|
||||
return fmt.Errorf("esbuild for %s: %s", a.Name, why)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := step(plain); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := step(launch, "--out-extension:.js=.mjs"); err != nil {
|
||||
return err
|
||||
}
|
||||
// Plain `.js` output is an ES module; said once.
|
||||
if err := os.WriteFile(filepath.Join(dir, final, "package.json"), []byte(`{"type":"module","private":true}`+"\n"), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, l := range launchers {
|
||||
path := filepath.Join(dir, final, filepath.FromSlash(l))
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
if err := os.Chmod(path, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeLaunchers writes, beside every entrypoint, an executable that imports the entrypoint and serves what it
|
||||
// registered over MCP on stdio (builder.go:1926-1956; novox/hq ADR 0193). Its answer is each entrypoint's launcher,
|
||||
// relative to the bundle's root.
|
||||
func writeLaunchers(root string, a artifact) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for _, entry := range a.Entrypoints {
|
||||
if !strings.HasSuffix(entry, ".js") {
|
||||
continue
|
||||
}
|
||||
launcher := strings.TrimSuffix(entry, ".js") + launcherSuffix
|
||||
body := "#!/usr/bin/env node\n" +
|
||||
"// Written by the mesh's builder (novox/hq ADR 0193): serve what " + entry + " registers,\n" +
|
||||
"// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.\n" +
|
||||
"import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\n" +
|
||||
"await import(\"./" + filepath.Base(entry) + "\");\n" +
|
||||
"await serveRegisteredOverStdio();\n"
|
||||
path := filepath.Join(root, filepath.FromSlash(launcher))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chmod(path, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[entry] = launcher
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sourcesFor turns compiled entrypoints back into what to compile (builder.go:1735-1747).
|
||||
func sourcesFor(entrypoints []string, out, ext string) []string {
|
||||
sources := make([]string, 0, len(entrypoints))
|
||||
for _, e := range entrypoints {
|
||||
at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/")
|
||||
sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+ext)
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
func sortedValues(m map[string]string) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for _, v := range m {
|
||||
out = append(out, v)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// readArtifact reads the named artifact of the module.json in dir, which must be a TypeScript bundle with entrypoints.
|
||||
func readArtifact(dir, name string) (artifact, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
|
||||
if err != nil {
|
||||
return artifact{}, fmt.Errorf("the module has no module.json: %w", err)
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return artifact{}, fmt.Errorf("module.json is not JSON: %w", err)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Name != name {
|
||||
continue
|
||||
}
|
||||
if a.Language != "typescript" {
|
||||
return artifact{}, fmt.Errorf("artifact %q is %q, and this module builds typescript", name, a.Language)
|
||||
}
|
||||
if len(a.Entrypoints) == 0 {
|
||||
return artifact{}, fmt.Errorf("artifact %q names no entrypoints", name)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
return artifact{}, fmt.Errorf("module.json declares no artifact %q", name)
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// realMachine installs the real toolchain from the module's own lock (npm ci --ignore-scripts, from the mesh's
|
||||
// registry for @novox, which reads without a credential) and makes two workspaces holding one fixture bundle.
|
||||
func realMachine(t *testing.T) (*Module, string) {
|
||||
t.Helper()
|
||||
for _, p := range []string{"node", "npm"} {
|
||||
if _, err := exec.LookPath(p); err != nil {
|
||||
t.Skipf("NOT RUN: no %s installed where this check runs (the mesh's Go toolchain image holds none); proven live on a build node", p)
|
||||
}
|
||||
}
|
||||
home, cache, work := t.TempDir(), t.TempDir(), t.TempDir()
|
||||
for _, n := range []string{"package.json", "package-lock.json"} {
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, "toolchain", n))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write(t, filepath.Join(home, "toolchain", n), string(raw), 0o644)
|
||||
}
|
||||
lock, err := ReadRecipe(filepath.Join(home, "toolchain", "package-lock.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run := forgekey.ExecRunnerFor(0)
|
||||
m := &Module{Node: "ace", Recipe: lock, Work: work, Run: run, Ctx: context.Background(),
|
||||
Toolchain: &Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: run, Registry: func() (Registry, error) {
|
||||
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
|
||||
}}}
|
||||
if err := m.Toolchain.Ensure(context.Background(), m.Recipe.Version()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{"w1", "w2"} {
|
||||
for p, body := range map[string]string{
|
||||
"modules/m/module.json": `{"module":"m","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`,
|
||||
"modules/m/package.json": `{"name":"m","type":"module","scripts":{"test":"node -e \"process.exit(0)\""}}`,
|
||||
"modules/m/tools/index.ts": "import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\nexport const f: () => Promise<void> = serveRegisteredOverStdio as () => Promise<void>;\n",
|
||||
} {
|
||||
write(t, filepath.Join(work, id, "tree", filepath.FromSlash(p)), body, 0o644)
|
||||
}
|
||||
}
|
||||
return m, work
|
||||
}
|
||||
|
||||
// 1490/AC2: a fixture TypeScript bundle importing @novox/mesh-sdk, built twice by the real tsc and esbuild, has equal
|
||||
// digests, and its archive holds the bundled .js, the .serve.mjs launcher and package.json.
|
||||
func TestAFixtureBundleBuiltTwiceHasOneDigest(t *testing.T) {
|
||||
m, work := realMachine(t)
|
||||
var digests []string
|
||||
for _, id := range []string{"w1", "w2"} {
|
||||
out, err := m.build(context.Background(), map[string]any{"workspace": id, "module": "modules/m", "artifact": "tools"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(BuildResult)
|
||||
digests = append(digests, r.Digest)
|
||||
if want := []string{"package.json", "tools/index.js", "tools/index.serve.mjs"}; !slices.Equal(r.Files, want) {
|
||||
t.Errorf("files = %v, want %v", r.Files, want)
|
||||
}
|
||||
}
|
||||
if digests[0] != digests[1] {
|
||||
t.Errorf("digests differ: %v", digests)
|
||||
}
|
||||
if links := symlinksUnder(t, work); len(links) != 0 {
|
||||
t.Errorf("symlinks in the workspaces: %v", links)
|
||||
}
|
||||
}
|
||||
|
||||
// 1491: the real type-check and test script of the same fixture pass the unit level.
|
||||
func TestTheFixturePassesTheUnitLevel(t *testing.T) {
|
||||
m, _ := realMachine(t)
|
||||
out, err := m.test(context.Background(), map[string]any{"workspace": "w1", "module": "modules/m", "level": "unit"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := out.(*TestResult); !r.Passed || r.NoTests || len(r.Ran) != 2 {
|
||||
t.Errorf("result = %+v", r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const bundleManifest = `{"module":"m","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript",` +
|
||||
`"entrypoints":["tools/index.js"],"loads":["tools/index.js"]%s}]}}`
|
||||
|
||||
// esbuildEffect stands in for esbuild's output: for each entry `.mesh-build/<a>/<path>.js` it writes the bundled file
|
||||
// under --outdir, named as esbuild names it (the input without its extension, then .js or .mjs when --out-extension:.js=.mjs is given).
|
||||
func esbuildEffect(t *testing.T) func(c call) {
|
||||
return func(c call) {
|
||||
if filepath.Base(c.name) != "esbuild" {
|
||||
return
|
||||
}
|
||||
var outdir, outbase, ext string
|
||||
var entries []string
|
||||
for _, a := range c.args {
|
||||
switch {
|
||||
case strings.HasPrefix(a, "--outdir="):
|
||||
outdir = strings.TrimPrefix(a, "--outdir=")
|
||||
case strings.HasPrefix(a, "--outbase="):
|
||||
outbase = strings.TrimPrefix(a, "--outbase=")
|
||||
case a == "--out-extension:.js=.mjs":
|
||||
ext = ".mjs"
|
||||
case !strings.HasPrefix(a, "--"):
|
||||
entries = append(entries, a)
|
||||
}
|
||||
}
|
||||
for _, e := range entries {
|
||||
rel := strings.TrimPrefix(strings.TrimPrefix(e, outbase), "/")
|
||||
// esbuild names an output after its input without the extension, and adds .js or the one asked for.
|
||||
rel = strings.TrimSuffix(rel, filepath.Ext(rel))
|
||||
if ext != "" {
|
||||
rel += ext
|
||||
} else {
|
||||
rel += ".js"
|
||||
}
|
||||
write(t, filepath.Join(c.dir, outdir, rel), "bundled "+rel, 0o644)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func buildArgs() map[string]any {
|
||||
return map[string]any{"workspace": "w1", "module": "modules/m", "artifact": "tools"}
|
||||
}
|
||||
|
||||
func builtMachine(t *testing.T, extra string) (*Module, *fakeMachine, string) {
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"modules/m/module.json": strings.Replace(bundleManifest, "%s", extra, 1),
|
||||
"modules/m/tools/index.ts": "export {}"})
|
||||
f.effect = esbuildEffect(t)
|
||||
return m, f, work
|
||||
}
|
||||
|
||||
// 1490/AC1: tsc and esbuild run with exactly the build agent's flags. The fixture is copied from mesh-controller
|
||||
// origin/main: internal/builder/toolchain.go:143-147 (tsc: --module NodeNext --moduleResolution NodeNext --target
|
||||
// ES2022 --rootDir .), builder.go:1637-1705 (--outDir .mesh-build/<artifact> and the sources) and :1971-2039 (esbuild:
|
||||
// --bundle --platform=node --format=esm --target=node22 --outbase --outdir --log-level=warning, the createRequire
|
||||
// banner, --out-extension:.js=.mjs for the launchers, --external:x). Where the build agent ran them in the toolchain
|
||||
// image they run here with the toolchain's real paths, from the module's directory (env -C).
|
||||
func TestTscAndEsbuildRunWithTheBuildAgentsFlags(t *testing.T) {
|
||||
m, f, work := builtMachine(t, `,"external":["pg"]`)
|
||||
if _, err := m.build(context.Background(), buildArgs()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Join(work, "w1", "stage-tools", "modules", "module")
|
||||
nm := filepath.Join(m.toolchainDir(), "node_modules")
|
||||
banner := "--banner:js=import { createRequire as __meshRequire } from 'node:module'; const require = __meshRequire(import.meta.url);"
|
||||
common := []string{"--bundle", "--platform=node", "--format=esm", "--target=node22",
|
||||
"--outbase=.mesh-build/tools", "--outdir=.mesh-build/tools.bundled", "--log-level=warning", banner, "--external:pg"}
|
||||
want := []string{
|
||||
"node " + nm + "/typescript/bin/tsc --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . --outDir .mesh-build/tools tools/index.ts",
|
||||
nm + "/esbuild/bin/esbuild .mesh-build/tools/tools/index.js " + strings.Join(common, " "),
|
||||
nm + "/esbuild/bin/esbuild .mesh-build/tools/tools/index.serve.mjs " + strings.Join(common, " ") + " --out-extension:.js=.mjs",
|
||||
}
|
||||
if len(f.calls) != len(want) {
|
||||
t.Fatalf("%d commands ran, want %d: %v", len(f.calls), len(want), f.calls)
|
||||
}
|
||||
for i, c := range f.calls {
|
||||
if c.dir != dir || joined(c) != want[i] {
|
||||
t.Errorf("command %d: in %s ran\n%s\nwant in %s\n%s", i, c.dir, joined(c), dir, want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readArchive(t *testing.T, path string) map[string]os.FileMode {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tr := tar.NewReader(zr)
|
||||
out := map[string]os.FileMode{}
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return out
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[h.Name] = os.FileMode(h.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
// 1490: the step packs the bundle: the archive holds the bundled .js, the .serve.mjs launcher (executable) and the
|
||||
// package.json that makes it an ES module, and the result is the archive's digest, the same for the same build.
|
||||
func TestTheBundleIsPackedWithItsLauncherAndPackageJson(t *testing.T) {
|
||||
m, _, _ := builtMachine(t, ``)
|
||||
out, err := m.build(context.Background(), buildArgs())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(BuildResult)
|
||||
if !strings.HasPrefix(r.Digest, "sha256:") || r.Bytes == 0 {
|
||||
t.Errorf("result = %+v", r)
|
||||
}
|
||||
if want := []string{"package.json", "tools/index.js", "tools/index.serve.mjs"}; !slices.Equal(r.Files, want) {
|
||||
t.Errorf("files = %v, want %v", r.Files, want)
|
||||
}
|
||||
modes := readArchive(t, r.Archive)
|
||||
if len(modes) != 3 || modes["tools/index.serve.mjs"]&0o111 == 0 {
|
||||
t.Errorf("archive = %v, want the three files with the launcher executable", modes)
|
||||
}
|
||||
again, err := m.build(context.Background(), buildArgs())
|
||||
if err != nil || again.(BuildResult).Digest != r.Digest {
|
||||
t.Errorf("a second build: %v, digest %v, want %s", err, again, r.Digest)
|
||||
}
|
||||
launcher, _ := os.ReadFile(filepath.Join(filepath.Dir(r.Archive), "stage-tools", "modules", "module", ".mesh-build", "tools", "tools", "index.serve.mjs"))
|
||||
if !strings.Contains(string(launcher), `import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";`) || !strings.Contains(string(launcher), `await import("./index.js");`) {
|
||||
t.Errorf("launcher = %q", launcher)
|
||||
}
|
||||
}
|
||||
|
||||
// 1490/AC3: no symlink is made anywhere in the workspace, with externals (whose packages are copied) or without.
|
||||
func TestNoSymlinkIsMadeInTheWorkspace(t *testing.T) {
|
||||
for _, extra := range []string{``, `,"external":["nats"]`} {
|
||||
m, _, work := builtMachine(t, extra)
|
||||
if _, err := m.build(context.Background(), buildArgs()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if links := symlinksUnder(t, filepath.Join(work, "w1")); len(links) != 0 {
|
||||
t.Errorf("symlinks in the workspace: %v", links)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1490: the bundle's SDK comes from the staging directory's node_modules, two levels above the module, and the packed bundle holds
|
||||
// the toolchain's packages only when the artifact keeps one external, never typescript or esbuild.
|
||||
func TestThePackagesAreCopiedBesideTheTreeAndIntoTheBundleOnlyForExternals(t *testing.T) {
|
||||
m, _, work := builtMachine(t, ``)
|
||||
out, err := m.build(context.Background(), buildArgs())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nm := filepath.Join(work, "w1", "stage-tools", "node_modules")
|
||||
for _, p := range []string{"@novox/mesh-sdk/dist/stdio.js", "@types/node/index.d.ts", "nats/lib/nats.js"} {
|
||||
if _, err := os.Stat(filepath.Join(nm, p)); err != nil {
|
||||
t.Errorf("the workspace's node_modules lacks %s", p)
|
||||
}
|
||||
}
|
||||
for _, p := range []string{"typescript", "esbuild", "@esbuild"} {
|
||||
if _, err := os.Stat(filepath.Join(nm, p)); err == nil {
|
||||
t.Errorf("the toolchain's own %s was copied", p)
|
||||
}
|
||||
}
|
||||
for f := range readArchive(t, out.(BuildResult).Archive) {
|
||||
if strings.HasPrefix(f, "node_modules") {
|
||||
t.Errorf("a bundle without externals carries %s", f)
|
||||
}
|
||||
}
|
||||
m, _, _ = builtMachine(t, `,"external":["nats"]`)
|
||||
out, _ = m.build(context.Background(), buildArgs())
|
||||
files := out.(BuildResult).Files
|
||||
if !slices.Contains(files, "node_modules/nats/lib/nats.js") || !slices.Contains(files, "node_modules/@novox/mesh-sdk/dist/stdio.js") {
|
||||
t.Errorf("files = %v, want the runtime packages beside the bundle", files)
|
||||
}
|
||||
for _, f := range files {
|
||||
if strings.Contains(f, "typescript") || strings.Contains(f, "esbuild") {
|
||||
t.Errorf("the bundle carries %s", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedCompilerFailsTheStepNamingIt(t *testing.T) {
|
||||
m, f, _ := builtMachine(t, ``)
|
||||
f.answer["tsc"] = forgekey.Ran{Status: 2, Stdout: tscTypeError}
|
||||
if _, err := m.build(context.Background(), buildArgs()); err == nil || !strings.Contains(err.Error(), "tsc for tools") || !strings.Contains(err.Error(), "tools/bad.ts") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyATypeScriptBundleOfTheModuleIsBuilt(t *testing.T) {
|
||||
m, _, _ := builtMachine(t, ``)
|
||||
for _, a := range []string{"nope", ""} {
|
||||
args := buildArgs()
|
||||
args["artifact"] = a
|
||||
if _, err := m.build(context.Background(), args); err == nil {
|
||||
t.Errorf("artifact %q was built", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1490 (review 7): the build agent runs a module mounted at /app/modules/module with the toolchain's packages at
|
||||
// /app/node_modules (mesh-controller builder.go `within`), so esbuild writes the path comments of what it inlines
|
||||
// relative to the module's directory as `../../node_modules/...`. The step builds at that depth: in a staging
|
||||
// directory whose module directory has the packages two levels above it, whatever the module's place in the
|
||||
// repository, so the bundle's bytes (and its digest) equal the build agent's.
|
||||
func TestTheBuildRunsWithThePackagesTwoLevelsAboveTheModule(t *testing.T) {
|
||||
for _, module := range []string{"modules/m", "."} {
|
||||
m, f, work := newMachine(t)
|
||||
files := map[string]string{"module.json": strings.Replace(bundleManifest, "%s", "", 1), "tools/index.ts": "export {}"}
|
||||
if module != "." {
|
||||
files = map[string]string{"modules/m/module.json": strings.Replace(bundleManifest, "%s", "", 1), "modules/m/tools/index.ts": "export {}"}
|
||||
}
|
||||
workspaceWith(t, work, files)
|
||||
f.effect = esbuildEffect(t)
|
||||
args := buildArgs()
|
||||
args["module"] = module
|
||||
if _, err := m.build(context.Background(), args); err != nil {
|
||||
t.Fatalf("%s: %v", module, err)
|
||||
}
|
||||
for _, c := range f.calls {
|
||||
rel, err := filepath.Rel(c.dir, filepath.Join(c.dir, "..", "..", "node_modules"))
|
||||
if err != nil || rel != "../../node_modules" {
|
||||
t.Fatalf("%s: relative path %q (%v)", module, rel, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(c.dir, "..", "..", "node_modules", "@novox", "mesh-sdk", "dist", "stdio.js")); err != nil {
|
||||
t.Errorf("%s: %s ran in %s, which has no packages at ../../node_modules: %v", module, filepath.Base(c.name), c.dir, err)
|
||||
}
|
||||
if filepath.Base(c.dir) != "module" || filepath.Base(filepath.Dir(c.dir)) != "modules" {
|
||||
t.Errorf("%s: ran in %s, want .../modules/module", module, c.dir)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(f.calls[0].dir, filepath.Join(work, "w1")) {
|
||||
t.Errorf("%s: staged outside the workspace: %s", module, f.calls[0].dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package main
|
||||
|
||||
// The test step (novox/hq ADR 0445 §4; issues 1491 and 1292): the two levels of a TypeScript package's tests, run
|
||||
// against a fetched workspace with the toolchain's tsc. The unit level is the package's type-check and its
|
||||
// package.json `test` script; the integration level is its `test:integration` script, with the type-check as its
|
||||
// setup. Each is timed and judged against its bound; a level over its bound says so in the result and does not fail
|
||||
// (the measure of the catalogue's TypeScript is a later release's, R4f). A package with neither a script nor .ts files
|
||||
// has no tests, and the result says so.
|
||||
//
|
||||
// **The bounds are the Go module's** (5 s for the unit level; 30 s of setup and 60 s of running for the integration
|
||||
// level), because ADR 0445 names no figures for TypeScript. They are constants here, said in every result.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// The bounds of a level (ADR 0445 §4).
|
||||
const (
|
||||
UnitBound = 5 * time.Second
|
||||
IntegrationSetupBound = 30 * time.Second
|
||||
IntegrationRunBound = 60 * time.Second
|
||||
)
|
||||
|
||||
// TestResult is what a test step passes with, with the field names of mesh-build-go's (issue 1290). The step ends
|
||||
// passed whenever the level ran, whatever its verdict, and Passed is the verdict; it ends failed only when the tests
|
||||
// could not be run (no toolchain, a command that cannot start, a workspace that was never fetched).
|
||||
type TestResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Module string `json:"module"`
|
||||
Level string `json:"level"`
|
||||
Passed bool `json:"passed"`
|
||||
// Why says, in words, why the level failed or went over its bound, or that it has no tests.
|
||||
Why string `json:"why,omitempty"`
|
||||
NoTests bool `json:"no_tests,omitempty"`
|
||||
// Ran names what ran: "type-check", "test" or "test:integration".
|
||||
Ran []string `json:"ran,omitempty"`
|
||||
// Elapsed is the level's running time in seconds (at the unit level the type-check and the test script together);
|
||||
// Setup is the integration level's setup, the type-check, in seconds. Bound and SetupBound are the bounds the
|
||||
// level is judged by.
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
Setup float64 `json:"setup,omitempty"`
|
||||
Bound float64 `json:"bound"`
|
||||
SetupBound float64 `json:"setup_bound,omitempty"`
|
||||
// OverBound says a bound was exceeded. Unlike mesh-build-go's, an over-bound level still passes: the measure of
|
||||
// the catalogue's TypeScript against the bounds is a later release's (R4f), not this one's gate.
|
||||
OverBound bool `json:"over_bound"`
|
||||
Failed []Failure `json:"failed,omitempty"`
|
||||
}
|
||||
|
||||
// Failure is a check or script that failed; Output is the first lines it wrote.
|
||||
type Failure struct {
|
||||
Package string `json:"package"`
|
||||
Test string `json:"test,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Module) testTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_node_test",
|
||||
Description: "Run a TypeScript package's tests in a fetched workspace: the unit level is the type-check (the toolchain's " +
|
||||
"tsc) and the package's test script, the integration level its test:integration script. Each level is timed " +
|
||||
"and judged against its bound (unit 5 s; integration 30 s setup and 60 s running); a level over its bound " +
|
||||
"says so and does not fail. A package with neither script nor .ts files has no tests. Answers a step id at " +
|
||||
"once; the step ends on its key in the steps state.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch step answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the package's directory in the workspace, '.' when it is the root"},
|
||||
"level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}},
|
||||
},
|
||||
"required": []string{"workspace", "level"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.test,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
level, _ := args["level"].(string)
|
||||
if level != "unit" && level != "integration" {
|
||||
return nil, fmt.Errorf("level %q is neither unit nor integration", level)
|
||||
}
|
||||
root, dir, err := m.moduleDirOf(workspace, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scripts, err := packageScripts(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sources, err := typeScriptFiles(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &TestResult{Workspace: workspace, Module: module, Level: level}
|
||||
script := "test"
|
||||
if level == "integration" {
|
||||
script = "test:integration"
|
||||
}
|
||||
hasScript := scripts[script] != ""
|
||||
if level == "integration" && !hasScript {
|
||||
res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test:integration script"
|
||||
return res, nil
|
||||
}
|
||||
if !hasScript && len(sources) == 0 {
|
||||
res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test script and no .ts files"
|
||||
return res, nil
|
||||
}
|
||||
if err := m.provide(root); err != nil {
|
||||
return nil, fmt.Errorf("providing the toolchain's packages to the workspace: %w", err)
|
||||
}
|
||||
where := module
|
||||
if where == "" {
|
||||
where = "."
|
||||
}
|
||||
var check, run time.Duration
|
||||
var failed []Failure
|
||||
// a command that could not start is not a verdict: the step fails. One that ran and said no is.
|
||||
ran := func(name string, r forgekey.Ran) error {
|
||||
if r.Err != "" {
|
||||
return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr))
|
||||
}
|
||||
if r.Status != 0 {
|
||||
failed = append(failed, Failure{Package: where, Test: name, Output: lines(r.Stdout + "\n" + r.Stderr)})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(sources) > 0 {
|
||||
start := m.now()
|
||||
r := m.runIn(ctx, dir, nil, "node", m.typeCheck(dir, sources)...)
|
||||
check = m.now().Sub(start)
|
||||
if err := ran("type-check", r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Ran = append(res.Ran, "type-check")
|
||||
}
|
||||
if hasScript && len(failed) == 0 {
|
||||
start := m.now()
|
||||
env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm")}
|
||||
r := m.runIn(ctx, dir, env, "npm", scriptArgs(script)...)
|
||||
run = m.now().Sub(start)
|
||||
if err := ran(script, r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Ran = append(res.Ran, script)
|
||||
}
|
||||
res.Failed = failed
|
||||
var why []string
|
||||
for _, f := range failed {
|
||||
why = append(why, fmt.Sprintf("%s failed in %s: %s", f.Test, f.Package, firstLine(f.Output)))
|
||||
}
|
||||
if level == "unit" {
|
||||
res.Elapsed, res.Bound = (check + run).Seconds(), UnitBound.Seconds()
|
||||
res.OverBound = check+run > UnitBound
|
||||
if res.OverBound {
|
||||
why = append(why, fmt.Sprintf("the unit level took %.2f s, over its bound of %s", res.Elapsed, UnitBound))
|
||||
}
|
||||
} else {
|
||||
res.Setup, res.Elapsed = check.Seconds(), run.Seconds()
|
||||
res.SetupBound, res.Bound = IntegrationSetupBound.Seconds(), IntegrationRunBound.Seconds()
|
||||
res.OverBound = check > IntegrationSetupBound || run > IntegrationRunBound
|
||||
if res.OverBound {
|
||||
why = append(why, fmt.Sprintf("the integration level took %.2f s of setup and %.2f s of running, over its bounds of %s and %s",
|
||||
res.Setup, res.Elapsed, IntegrationSetupBound, IntegrationRunBound))
|
||||
}
|
||||
}
|
||||
res.Passed = len(failed) == 0
|
||||
res.Why = strings.Join(why, "; ")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// lines is the first lines of a command's output.
|
||||
func lines(out string) string {
|
||||
var keep []string
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
keep = append(keep, l)
|
||||
if len(keep) == 15 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(keep, "\n")
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
if i := strings.IndexByte(strings.TrimSpace(s), '\n'); i >= 0 {
|
||||
return strings.TrimSpace(s)[:i]
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// scriptArgs is npm's line for a script: `npm test` for test, `npm run <name>` for another.
|
||||
func scriptArgs(script string) []string {
|
||||
if script == "test" {
|
||||
return []string{"test"}
|
||||
}
|
||||
return []string{"run", script}
|
||||
}
|
||||
|
||||
// typeCheck is tsc's line for the type-check: the package's tsconfig.json when it has one, else the build agent's
|
||||
// flags (toolchain.go:143-147) with --noEmit over every .ts file of the package.
|
||||
func (m *Module) typeCheck(dir string, sources []string) []string {
|
||||
if _, err := os.Stat(filepath.Join(dir, "tsconfig.json")); err == nil {
|
||||
return []string{m.tsc(), "--noEmit", "-p", "."}
|
||||
}
|
||||
return append([]string{m.tsc(), "--noEmit", "--module", "NodeNext", "--moduleResolution", "NodeNext",
|
||||
"--target", "ES2022", "--rootDir", "."}, sources...)
|
||||
}
|
||||
|
||||
// packageScripts are the scripts of the package.json in dir (none without one).
|
||||
func packageScripts(dir string) (map[string]string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
||||
if os.IsNotExist(err) {
|
||||
return map[string]string{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var p struct {
|
||||
Scripts map[string]string `json:"scripts"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return nil, fmt.Errorf("the package.json is not JSON: %w", err)
|
||||
}
|
||||
return p.Scripts, nil
|
||||
}
|
||||
|
||||
// typeScriptFiles are the .ts files of the package (declaration files excepted), relative and sorted, without
|
||||
// node_modules, hidden directories and a build's output.
|
||||
func typeScriptFiles(dir string) ([]string, error) {
|
||||
var out []string
|
||||
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := d.Name()
|
||||
if d.IsDir() {
|
||||
if path != dir && (name == "node_modules" || strings.HasPrefix(name, ".")) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(name, ".ts") && !strings.HasSuffix(name, ".d.ts") {
|
||||
rel, _ := filepath.Rel(dir, path)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// tscTypeError is what tsc 5.9.3 printed for `export const n: number = "text";` in tools/bad.ts, run on this machine
|
||||
// on 2026-10-20 with the flags of the type-check below (status 2).
|
||||
const tscTypeError = "tools/bad.ts(1,14): error TS2322: Type 'string' is not assignable to type 'number'.\n"
|
||||
|
||||
func testArgs(level string) map[string]any {
|
||||
return map[string]any{"workspace": "w1", "module": ".", "level": level}
|
||||
}
|
||||
|
||||
// 1491/AC1: a package with a type error fails the unit level, naming the file.
|
||||
func TestATypeErrorFailsTheUnitLevelNamingTheFile(t *testing.T) {
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"tools/bad.ts": `export const n: number = "text";`, "package.json": `{"name":"x"}`})
|
||||
f.answer["tsc"] = forgekey.Ran{Status: 2, Stdout: tscTypeError}
|
||||
out, err := m.test(context.Background(), testArgs("unit"))
|
||||
if err != nil {
|
||||
t.Fatalf("a type error ended the step %v, want it finished with the verdict in its result", err)
|
||||
}
|
||||
r := out.(*TestResult)
|
||||
if r.Passed || len(r.Failed) != 1 || !strings.Contains(r.Why, "tools/bad.ts") || !strings.Contains(r.Failed[0].Output, "TS2322") {
|
||||
t.Fatalf("result = %+v, want passed false, naming tools/bad.ts", r)
|
||||
}
|
||||
if c := f.calls[0]; !strings.Contains(joined(c), "--noEmit") || !strings.Contains(joined(c), "typescript/bin/tsc") ||
|
||||
!strings.Contains(joined(c), "--moduleResolution NodeNext --target ES2022") || !strings.HasSuffix(joined(c), "tools/bad.ts") {
|
||||
t.Errorf("the type-check ran %s", joined(c))
|
||||
}
|
||||
}
|
||||
|
||||
// 1491/AC2: a package with a passing test script passes the unit level, which reports its time; over its bound it
|
||||
// says so and still passes.
|
||||
func TestAPassingTestScriptPassesTheUnitLevelAndReportsItsTime(t *testing.T) {
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "package.json": `{"scripts":{"test":"node --test"}}`})
|
||||
f.cost["tsc"], f.cost["npm"] = 400*time.Millisecond, 600*time.Millisecond
|
||||
out, err := m.test(context.Background(), testArgs("unit"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(*TestResult)
|
||||
if !r.Passed || r.NoTests || r.Elapsed != 1 || r.OverBound || r.Bound != 5 || strings.Join(r.Ran, ",") != "type-check,test" {
|
||||
t.Errorf("result = %+v, want the type-check and the test script in 1 s within the bound", r)
|
||||
}
|
||||
last := f.calls[len(f.calls)-1]
|
||||
if last.name != "npm" || strings.Join(last.args, " ") != "test" {
|
||||
t.Errorf("ran %s, want npm test", joined(last))
|
||||
}
|
||||
f.cost["npm"] = 6 * time.Second
|
||||
out, err = m.test(context.Background(), testArgs("unit"))
|
||||
if r := out.(*TestResult); err != nil || !r.OverBound {
|
||||
t.Errorf("a slow unit level = %+v, %v, want passed and over_bound", r, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1491/AC3: a package with neither script nor .ts files has no tests, and nothing runs.
|
||||
func TestAPackageWithNeitherScriptNorTypeScriptHasNoTests(t *testing.T) {
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"package.json": `{"name":"x"}`, "node_modules/p/x.ts": "ignored", "types.d.ts": "declare const a: 1;"})
|
||||
for _, level := range []string{"unit", "integration"} {
|
||||
out, err := m.test(context.Background(), testArgs(level))
|
||||
r := out.(*TestResult)
|
||||
if err != nil || !r.Passed || !r.NoTests || !strings.Contains(r.Why, "no tests") {
|
||||
t.Errorf("%s = %+v, %v, want the result to say no tests", level, r, err)
|
||||
}
|
||||
}
|
||||
if len(f.calls) != 0 {
|
||||
t.Errorf("%d commands ran for a package with no tests", len(f.calls))
|
||||
}
|
||||
}
|
||||
|
||||
// 1491/AC4: with test:integration present the integration level runs it, and reports the setup (the type-check) and
|
||||
// the running time apart.
|
||||
func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testing.T) {
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "tsconfig.json": `{}`,
|
||||
"package.json": `{"scripts":{"test":"x","test:integration":"node --test int"}}`})
|
||||
f.cost["tsc"], f.cost["npm"] = 2*time.Second, 40*time.Second
|
||||
out, err := m.test(context.Background(), testArgs("integration"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(*TestResult)
|
||||
if r.Setup != 2 || r.Elapsed != 40 || r.SetupBound != 30 || r.Bound != 60 || !r.Passed || r.OverBound || r.Level != "integration" {
|
||||
t.Errorf("result = %+v, want setup 2 s and running 40 s apart", r)
|
||||
}
|
||||
last := f.calls[len(f.calls)-1]
|
||||
if strings.Join(last.args, " ") != "run test:integration" {
|
||||
t.Errorf("ran %s, want npm run test:integration", joined(last))
|
||||
}
|
||||
if tsc := f.calls[0]; strings.Join(tsc.args[1:], " ") != "--noEmit -p ." {
|
||||
t.Errorf("a package with a tsconfig was checked with %s", joined(tsc))
|
||||
}
|
||||
f.cost["npm"] = 61 * time.Second
|
||||
out, _ = m.test(context.Background(), testArgs("integration"))
|
||||
if !out.(*TestResult).OverBound {
|
||||
t.Errorf("61 s of running is within the bound")
|
||||
}
|
||||
}
|
||||
|
||||
// The commands of a step get an environment built from scratch: no word of this process's.
|
||||
func TestTheStepsCommandsGetAnEnvironmentOfTheirOwn(t *testing.T) {
|
||||
t.Setenv("MESH_SECRET_TOKEN", "hunter2")
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "package.json": `{"scripts":{"test":"x"}}`})
|
||||
if _, err := m.test(context.Background(), testArgs("unit")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range f.calls {
|
||||
for _, e := range c.env {
|
||||
if strings.Contains(e, "hunter2") || strings.HasPrefix(e, "MESH_") {
|
||||
t.Errorf("%s was given %s", joined(c), e)
|
||||
}
|
||||
}
|
||||
if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") {
|
||||
t.Errorf("%s env = %v, want PATH, HOME and the locale", c.name, c.env)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1491: the step finishes with the verdict when the tests fail, and fails only when they could not be run: a failing
|
||||
// test script is a verdict, a program that cannot start is not.
|
||||
func TestAFailingScriptIsAVerdictAndACommandThatCannotStartIsAFailedStep(t *testing.T) {
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"package.json": `{"scripts":{"test":"x"}}`})
|
||||
f.answer["npm"] = forgekey.Ran{Status: 1, Stdout: "not ok 1 - adds\n"}
|
||||
out, err := m.test(context.Background(), testArgs("unit"))
|
||||
r, _ := out.(*TestResult)
|
||||
if err != nil || r == nil || r.Passed || len(r.Failed) != 1 || r.Failed[0].Test != "test" || !strings.Contains(r.Failed[0].Output, "not ok 1") {
|
||||
t.Errorf("a failing script = %+v, %v, want passed false and its output", r, err)
|
||||
}
|
||||
f.answer["npm"] = forgekey.Ran{Status: 127, Err: "ENOENT"}
|
||||
if out, err := m.test(context.Background(), testArgs("unit")); err == nil {
|
||||
t.Errorf("npm not installed = %v, want the step to fail", out)
|
||||
}
|
||||
if _, err := m.test(context.Background(), map[string]any{"workspace": "nope", "level": "unit"}); err == nil {
|
||||
t.Errorf("a workspace never fetched did not fail the step")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWorkspaceOutsideTheWorkDirectoryIsRefused(t *testing.T) {
|
||||
m, _, _ := newMachine(t)
|
||||
for _, a := range []map[string]any{
|
||||
{"workspace": "../etc", "level": "unit"},
|
||||
{"workspace": "w1", "module": "../x", "level": "unit"},
|
||||
{"workspace": "w1", "module": "/etc", "level": "unit"},
|
||||
{"workspace": "w1", "level": "smoke"},
|
||||
} {
|
||||
if _, err := m.test(context.Background(), a); err == nil {
|
||||
t.Errorf("%v was accepted", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1491 (review 6): --no-bin-links leaves the toolchain no bin directory, so `npm run build` found no tsc or esbuild
|
||||
// and exited 127, which read as a failed test. The workspace now has a bin directory of small shell scripts (files,
|
||||
// not links) that call the toolchain's tsc and esbuild, first on the PATH of the test script.
|
||||
func TestTheTestScriptFindsTscAndEsbuildOnItsPath(t *testing.T) {
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"package.json": `{"scripts":{"test":"tsc && esbuild"}}`})
|
||||
if _, err := m.test(context.Background(), testArgs("unit")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bin := filepath.Join(work, "w1", "bin")
|
||||
nm := filepath.Join(m.toolchainDir(), "node_modules")
|
||||
for name, want := range map[string]string{
|
||||
"tsc": "exec node '" + nm + "/typescript/bin/tsc' \"$@\"",
|
||||
"esbuild": "exec '" + nm + "/esbuild/bin/esbuild' \"$@\"",
|
||||
} {
|
||||
info, err := os.Lstat(filepath.Join(bin, name))
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 {
|
||||
t.Fatalf("bin/%s: %v %v, want an executable file", name, info, err)
|
||||
}
|
||||
raw, _ := os.ReadFile(filepath.Join(bin, name))
|
||||
if !strings.HasPrefix(string(raw), "#!/bin/sh\n") || !strings.Contains(string(raw), want) {
|
||||
t.Errorf("bin/%s = %q, want a script calling the toolchain's program", name, raw)
|
||||
}
|
||||
}
|
||||
npm := f.calls[len(f.calls)-1]
|
||||
path := ""
|
||||
for _, e := range npm.env {
|
||||
if strings.HasPrefix(e, "PATH=") {
|
||||
path = strings.TrimPrefix(e, "PATH=") // the last PATH wins
|
||||
}
|
||||
}
|
||||
if npm.name != "npm" || !strings.HasPrefix(path, bin+":") {
|
||||
t.Errorf("npm ran with PATH %q, want %s first", path, bin)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issues 1483 and 1488 (for the indexer, issue 1444): the guard emits toolchain.reported after each write of the
|
||||
// toolchains key, with the node and the language.
|
||||
func TestTheGuardEmitsToolchainReportedAfterEachWrite(t *testing.T) {
|
||||
m := moduleWith(newStates(), "v26.10.0")
|
||||
type event struct {
|
||||
typ string
|
||||
body map[string]string
|
||||
}
|
||||
var got []event
|
||||
m.Emit = func(typ string, body any) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
var b map[string]string
|
||||
json.Unmarshal(raw, &b)
|
||||
got = append(got, event{typ, b})
|
||||
return nil
|
||||
}
|
||||
if err := m.Check(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].typ != "toolchain.reported" || got[0].body["node"] != "ace" || got[0].body["language"] != "typescript" || len(got[0].body) != 2 {
|
||||
t.Errorf("events = %+v, want one toolchain.reported for ace, typescript", got)
|
||||
}
|
||||
m.Check(context.Background())
|
||||
if len(got) != 2 {
|
||||
t.Errorf("a second guard pass emitted %d events in all, want 2", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// The event is carried by the module's stream, not named in emits (ADR 0363: one place); the validator checks that.
|
||||
func TestTheManifestDeclaresTheToolchainsStream(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Streams []struct {
|
||||
Name string `json:"name"`
|
||||
Events []string `json:"events"`
|
||||
} `json:"streams"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) != 1 || m.Streams[0].Name != "toolchains-node" || !slices.Equal(m.Streams[0].Events, []string{"toolchain.**"}) {
|
||||
t.Errorf("streams = %+v (%v)", m.Streams, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
package main
|
||||
|
||||
// The fetch step (novox/hq ADR 0405 §5, ADR 0466 §4 and §5, ADR 0469 §2 and §5; issue 1489): a commit of a repository
|
||||
// of the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and for each
|
||||
// TypeScript module in it the packages its package.json names beyond the SDK, installed from its committed lock.
|
||||
//
|
||||
// **Only what the module itself depends on is installed.** The SDK and nats come with the toolchain, so a bundle that
|
||||
// names only @novox/mesh-sdk (and build-time types and typescript) resolves nothing. One that names more must carry a
|
||||
// package-lock.json, or the step fails naming its directory. The install is the build agent's (mesh-controller
|
||||
// internal/builder/dependencies.go:49-95): in a scratch copy of package.json and the lock with the SDK, devDependencies
|
||||
// and scripts taken out, `npm ci --omit=dev --omit=peer --ignore-scripts`, the install's copies of the SDK removed, and
|
||||
// the result put at the module's node_modules. This module adds --no-bin-links, so the workspace holds no symlink.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1).
|
||||
const ForgeAlias = "mesh-forge"
|
||||
|
||||
// sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
|
||||
const sdkPackage = "@novox/mesh-sdk"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept.
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
|
||||
// FetchResult is what a passed fetch step says.
|
||||
type FetchResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Commit string `json:"commit"`
|
||||
// Modules are the TypeScript modules' directories found, and Installed those whose own packages were installed.
|
||||
Modules []string `json:"modules"`
|
||||
Installed []string `json:"installed"`
|
||||
// Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK
|
||||
// with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459).
|
||||
Refused []Refusal `json:"refused"`
|
||||
}
|
||||
|
||||
// Refusal is a module a fetch did not install the packages of.
|
||||
type Refusal struct {
|
||||
Module string `json:"module"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
func (m *Module) fetchTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_node_fetch",
|
||||
Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
|
||||
"workspace of its own, and install each TypeScript module's own packages from its lock (a module naming only the " +
|
||||
"SDK installs nothing). Answers a step id at once; the step ends on its key in the steps state with the workspace's id.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
||||
},
|
||||
"required": []string{"repository", "commit"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.fetch,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
root := filepath.Join(m.Work, id)
|
||||
tree := filepath.Join(root, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// git and ssh need PATH and the account's HOME, which a clean runner does not supply.
|
||||
gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
|
||||
return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
|
||||
}
|
||||
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
|
||||
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirs, err := m.typeScriptModules(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
|
||||
for _, d := range dirs {
|
||||
res.Modules = append(res.Modules, d)
|
||||
did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if refused != "" {
|
||||
res.Refused = append(res.Refused, Refusal{Module: d, Reason: refused})
|
||||
}
|
||||
if did {
|
||||
res.Installed = append(res.Installed, d)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to
|
||||
// it, in order.
|
||||
func (m *Module) typeScriptModules(ctx context.Context, tree string) ([]string, error) {
|
||||
r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
var dirs []string
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "" {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var mj struct {
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if json.Unmarshal(raw, &mj) != nil {
|
||||
continue // a module.json the build step will refuse by name, not a fetch's business
|
||||
}
|
||||
for _, a := range mj.Build.Artifacts {
|
||||
if a.Language == "typescript" {
|
||||
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(dirs)
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
// ownDependencies are the package.json's dependencies beyond the SDK and beyond what is build-time: the types and
|
||||
// typescript, which the toolchain carries (dependencies.go:49-70). Sorted; none for a module without a package.json.
|
||||
func ownDependencies(dir string) (names []string, pkg map[string]any, err error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &pkg); err != nil {
|
||||
return nil, nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
|
||||
}
|
||||
deps, _ := pkg["dependencies"].(map[string]any)
|
||||
for name := range deps {
|
||||
if name != sdkPackage && name != "typescript" && !strings.HasPrefix(name, "@types/") {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, pkg, nil
|
||||
}
|
||||
|
||||
// lockProblem says why a module's own packages cannot be installed: it names packages beyond the SDK and has no
|
||||
// package-lock.json. Empty when it names none or has its lock.
|
||||
func lockProblem(dir, rel string) (string, error) {
|
||||
names, _, err := ownDependencies(dir)
|
||||
if err != nil || len(names) == 0 {
|
||||
return "", err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
|
||||
return fmt.Sprintf("%s names packages beyond the SDK and has no package-lock.json (%s): commit one", rel, strings.Join(names, ", ")), nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// installOwn installs a module's own packages into its directory, or does nothing for a module that has none. A module
|
||||
// that names packages and has no lock is not installed and not an error: the reason is answered, the fetch passes and
|
||||
// the build of that bundle fails naming it.
|
||||
func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (installed bool, refused string, err error) {
|
||||
names, pkg, err := ownDependencies(dir)
|
||||
if err != nil || len(names) == 0 {
|
||||
return false, "", err
|
||||
}
|
||||
if why, err := lockProblem(dir, rel); why != "" || err != nil {
|
||||
return false, why, err
|
||||
}
|
||||
lock, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if m.Toolchain == nil || m.Toolchain.Registry == nil {
|
||||
return false, "", fmt.Errorf("the module has no package registry to install %s from", rel)
|
||||
}
|
||||
reg, err := m.Toolchain.Registry()
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
// A scratch copy, so the module's package.json and lock are never rewritten (dependencies.go installSteps).
|
||||
scratch := filepath.Join(root, "npm-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
|
||||
if err := os.MkdirAll(scratch, 0o700); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies"} {
|
||||
if d, ok := pkg[k].(map[string]any); ok {
|
||||
delete(d, sdkPackage)
|
||||
}
|
||||
}
|
||||
delete(pkg, "devDependencies")
|
||||
delete(pkg, "scripts")
|
||||
raw, _ := json.Marshal(pkg)
|
||||
if err := os.WriteFile(filepath.Join(scratch, "package.json"), raw, 0o600); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(scratch, "package-lock.json"), lock, 0o600); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
// The registry is given through a config file of its own, never on the command line, and removed after.
|
||||
npmrc := filepath.Join(scratch, ".npmrc-install")
|
||||
if err := os.WriteFile(npmrc, []byte(reg.Npmrc()), 0o600); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
env := append(m.baseEnv(), "npm_config_cache="+filepath.Join(m.Toolchain.Paths.Cache, "npm"))
|
||||
r := m.Run(ctx, env, "npm", "ci", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links",
|
||||
"--no-audit", "--no-fund", "--prefix", scratch, "--userconfig", npmrc)
|
||||
os.Remove(npmrc)
|
||||
if why := ranFailed(r); why != "" {
|
||||
return false, "", fmt.Errorf("installing the packages of %s (%s): %s", rel, strings.Join(names, ", "), why)
|
||||
}
|
||||
nm := filepath.Join(scratch, "node_modules")
|
||||
if err := removeSDKCopies(nm); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(dir, "node_modules")); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if err := os.Rename(nm, filepath.Join(dir, "node_modules")); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
return true, "", os.RemoveAll(scratch)
|
||||
}
|
||||
|
||||
// removeSDKCopies removes every copy of the SDK the install pulled in: the toolchain's is the pin.
|
||||
func removeSDKCopies(nm string) error {
|
||||
var found []string
|
||||
err := filepath.WalkDir(nm, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && strings.HasSuffix(filepath.ToSlash(p), "/node_modules/@novox/mesh-sdk") {
|
||||
found = append(found, p)
|
||||
return fs.SkipDir
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range found {
|
||||
if err := os.RemoveAll(p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const commitSHA = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
var fetchArgs = map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}
|
||||
|
||||
// fetchMachine is a machine whose git checks out the given files into the tree it is asked to fetch into (the fake
|
||||
// answers the commands of FetchCommit and `git ls-files`), and whose npm ci writes the packages the test names.
|
||||
func fetchMachine(t *testing.T, files map[string]string, installs map[string]string) (*Module, *fakeMachine) {
|
||||
t.Helper()
|
||||
m, f, _ := newMachine(t)
|
||||
f.answer["git rev-parse"] = forgekey.Ran{Stdout: commitSHA + "\n"}
|
||||
var manifests []string
|
||||
for p := range files {
|
||||
if strings.HasSuffix(p, "module.json") {
|
||||
manifests = append(manifests, p)
|
||||
}
|
||||
}
|
||||
f.answer["git ls-files"] = forgekey.Ran{Stdout: strings.Join(manifests, "\n") + "\n"}
|
||||
f.effect = func(c call) {
|
||||
switch {
|
||||
case c.name == "git" && slices_contains(c.args, "checkout"):
|
||||
for i, a := range c.args {
|
||||
if a == "-C" {
|
||||
for p, body := range files {
|
||||
write(t, filepath.Join(c.args[i+1], filepath.FromSlash(p)), body, 0o644)
|
||||
}
|
||||
}
|
||||
}
|
||||
case c.name == "npm":
|
||||
for i, a := range c.args {
|
||||
if a == "--prefix" {
|
||||
for p, body := range installs {
|
||||
write(t, filepath.Join(c.args[i+1], "node_modules", filepath.FromSlash(p)), body, 0o644)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return m, f
|
||||
}
|
||||
|
||||
func slices_contains(l []string, s string) bool {
|
||||
for _, x := range l {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func typeScriptModule(pkg string) map[string]string {
|
||||
return map[string]string{
|
||||
"modules/m/module.json": `{"module":"m","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`,
|
||||
"modules/m/package.json": pkg,
|
||||
}
|
||||
}
|
||||
|
||||
func npmCalls(f *fakeMachine) []call {
|
||||
var out []call
|
||||
for _, c := range f.calls {
|
||||
if c.name == "npm" {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 1489/AC1: a module whose package.json names only @novox/mesh-sdk and @types/node (and typescript) installs nothing
|
||||
// and the step passes.
|
||||
func TestAModuleNamingOnlyTheSdkAndTypesInstallsNothing(t *testing.T) {
|
||||
m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"},"devDependencies":{"@types/node":"^22","typescript":"^5"}}`), nil)
|
||||
out, err := m.fetch(context.Background(), fetchArgs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(FetchResult)
|
||||
if len(npmCalls(f)) != 0 || len(r.Installed) != 0 || len(r.Modules) != 1 || r.Modules[0] != "modules/m" || r.Commit != commitSHA {
|
||||
t.Errorf("result = %+v, npm calls %d, want the module found and nothing installed", r, len(npmCalls(f)))
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "module.json")); err != nil {
|
||||
t.Errorf("the commit is not checked out in the workspace: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1489/AC2: a module naming lodash with a lock gets npm ci --omit=dev --ignore-scripts (fake runner), its install put
|
||||
// at its node_modules without the SDK and without any copy of it; without a lock the step fails naming the directory.
|
||||
func TestAModuleNamingLodashWithALockIsInstalledWithoutTheSdkAndWithoutALockFails(t *testing.T) {
|
||||
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"},"devDependencies":{"x":"1"},"scripts":{"test":"y"}}`)
|
||||
files["modules/m/package-lock.json"] = `{"lockfileVersion":3}`
|
||||
m, f := fetchMachine(t, files, map[string]string{"lodash/index.js": "l", "@novox/mesh-sdk/index.js": "sdk", "a/node_modules/@novox/mesh-sdk/i.js": "nested"})
|
||||
out, err := m.fetch(context.Background(), fetchArgs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(FetchResult)
|
||||
calls := npmCalls(f)
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("npm ran %d times, want once", len(calls))
|
||||
}
|
||||
line := strings.Join(calls[0].args, " ")
|
||||
if !strings.HasPrefix(line, "ci --omit=dev --omit=peer --ignore-scripts --no-bin-links") {
|
||||
t.Errorf("ran npm %s", line)
|
||||
}
|
||||
nm := filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules")
|
||||
if _, err := os.Stat(filepath.Join(nm, "lodash", "index.js")); err != nil {
|
||||
t.Errorf("lodash is not at the module's node_modules: %v", err)
|
||||
}
|
||||
for _, gone := range []string{"@novox/mesh-sdk", "a/node_modules/@novox/mesh-sdk"} {
|
||||
if _, err := os.Stat(filepath.Join(nm, gone)); err == nil {
|
||||
t.Errorf("a copy of the SDK is left at %s", gone)
|
||||
}
|
||||
}
|
||||
if got := strings.Join(r.Installed, ","); got != "modules/m" {
|
||||
t.Errorf("installed = %q", got)
|
||||
}
|
||||
// the module's own package.json is untouched: the install worked on a scratch copy.
|
||||
if raw, _ := os.ReadFile(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "package.json")); !strings.Contains(string(raw), "scripts") {
|
||||
t.Errorf("the module's package.json was rewritten: %s", raw)
|
||||
}
|
||||
var scratch []string
|
||||
for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) {
|
||||
scratch = append(scratch, e)
|
||||
}
|
||||
if strings.Contains(strings.Join(scratch, ","), "npm-") {
|
||||
t.Errorf("the scratch install is left in the workspace: %v", scratch)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// 1489 (the 1459 settlement): a module naming packages with no package-lock.json does not fail the fetch: it passes,
|
||||
// installs nothing for that module, and its result lists the refused bundle with its reason; a build of that bundle
|
||||
// fails naming it.
|
||||
func TestAModuleWithoutALockIsRefusedInTheResultNotTheFetch(t *testing.T) {
|
||||
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"}}`)
|
||||
files["modules/ok/module.json"] = `{"module":"ok","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`
|
||||
files["modules/ok/package.json"] = `{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`
|
||||
m, f := fetchMachine(t, files, nil)
|
||||
out, err := m.fetch(context.Background(), fetchArgs)
|
||||
if err != nil {
|
||||
t.Fatalf("a module without a lock failed the fetch: %v", err)
|
||||
}
|
||||
r := out.(FetchResult)
|
||||
if len(npmCalls(f)) != 0 || len(r.Installed) != 0 || len(r.Modules) != 2 {
|
||||
t.Errorf("result = %+v, npm calls %d", r, len(npmCalls(f)))
|
||||
}
|
||||
if len(r.Refused) != 1 || r.Refused[0].Module != "modules/m" ||
|
||||
!strings.Contains(r.Refused[0].Reason, "names packages beyond the SDK and has no package-lock.json") || !strings.Contains(r.Refused[0].Reason, "lodash") {
|
||||
t.Fatalf("refused = %+v, want modules/m with its reason", r.Refused)
|
||||
}
|
||||
// the build of that bundle fails naming it, without running a compiler.
|
||||
f.calls = nil
|
||||
_, err = m.build(context.Background(), map[string]any{"workspace": r.Workspace, "module": "modules/m", "artifact": "tools"})
|
||||
if err == nil || !strings.Contains(err.Error(), "modules/m") || !strings.Contains(err.Error(), "no package-lock.json") || len(f.calls) != 0 {
|
||||
t.Errorf("build = %v after %d commands, want the refusal naming modules/m and nothing run", err, len(f.calls))
|
||||
}
|
||||
}
|
||||
|
||||
// 1489 (review 4): a bundle depending on another @novox package is installed, the mesh's registry given to npm ci
|
||||
// through --userconfig, as the toolchain's install gives it.
|
||||
func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *testing.T) {
|
||||
files := typeScriptModule(`{"dependencies":{"@novox/other":"^1","lodash":"^4"}}`)
|
||||
files["modules/m/package-lock.json"] = `{}`
|
||||
m, f := fetchMachine(t, files, map[string]string{"@novox/other/index.js": "o", "lodash/index.js": "l"})
|
||||
m.Toolchain.Registry = func() (Registry, error) {
|
||||
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil
|
||||
}
|
||||
var npmrc string
|
||||
inner := f.effect
|
||||
f.effect = func(c call) {
|
||||
inner(c)
|
||||
for i, a := range c.args {
|
||||
if c.name == "npm" && a == "--userconfig" {
|
||||
raw, _ := os.ReadFile(c.args[i+1])
|
||||
npmrc = string(raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
out, err := m.fetch(context.Background(), fetchArgs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || !strings.Contains(npmrc, ":_auth=") {
|
||||
t.Errorf("npm ci was given the config %q", npmrc)
|
||||
}
|
||||
if strings.Contains(strings.Join(npmCalls(f)[0].args, " "), "pw") {
|
||||
t.Errorf("the credential is on npm's command line")
|
||||
}
|
||||
r := out.(FetchResult)
|
||||
if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules", "@novox", "other", "index.js")); err != nil {
|
||||
t.Errorf("@novox/other was not installed: %v", err)
|
||||
}
|
||||
for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) {
|
||||
if strings.HasPrefix(e, "npm-") {
|
||||
t.Errorf("the scratch install (and its credential file) is left: %s", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustReadDir(t *testing.T, dir string) []string {
|
||||
t.Helper()
|
||||
es, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, e := range es {
|
||||
out = append(out, e.Name())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 1489/AC3: given a process environment holding a token, no command of the step sees it. The commands the step makes
|
||||
// are given PATH, HOME and the locale and their own words; and the real clean runner, run on `env` here, prints only
|
||||
// those.
|
||||
func TestNoCommandOfTheStepSeesATokenOfTheProcess(t *testing.T) {
|
||||
t.Setenv("MESH_BUS_CREDENTIAL", "hunter2")
|
||||
t.Setenv("NPM_TOKEN", "hunter3")
|
||||
files := typeScriptModule(`{"dependencies":{"lodash":"^4"}}`)
|
||||
files["modules/m/package-lock.json"] = `{}`
|
||||
m, f := fetchMachine(t, files, map[string]string{"lodash/index.js": "l"})
|
||||
if _, err := m.fetch(context.Background(), fetchArgs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.calls) < 6 {
|
||||
t.Fatalf("only %d commands ran", len(f.calls))
|
||||
}
|
||||
for _, c := range f.calls {
|
||||
for _, e := range c.env {
|
||||
if strings.Contains(e, "hunter") || strings.HasPrefix(e, "MESH_") || strings.HasPrefix(e, "NPM_TOKEN") {
|
||||
t.Errorf("%s %v was given %s", c.name, c.args[:min(len(c.args), 3)], e)
|
||||
}
|
||||
}
|
||||
if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") || c.env[2] != "LC_ALL=C" {
|
||||
t.Errorf("%s was given %v, want PATH, HOME and the locale first", c.name, c.env)
|
||||
}
|
||||
}
|
||||
// The runner main wires: what it hands a command is exactly what the step gave.
|
||||
ran := forgekey.ExecRunnerClean(10*time.Second)(context.Background(), append(m.baseEnv(), "ONLY=this"), "env")
|
||||
if ran.Status != 0 || strings.Contains(ran.Stdout, "hunter") || !strings.Contains(ran.Stdout, "ONLY=this") {
|
||||
t.Errorf("the clean runner printed %q", ran.Stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRefusesWhatIsNotARepositoryBeforeAnythingRuns(t *testing.T) {
|
||||
m, f := fetchMachine(t, nil, nil)
|
||||
for _, repo := range []string{"", "x", "../etc/passwd", "a/b/c", "a/..", "o/r;rm"} {
|
||||
if _, err := m.fetch(context.Background(), map[string]any{"repository": repo, "commit": commitSHA}); err == nil {
|
||||
t.Errorf("%q was accepted", repo)
|
||||
}
|
||||
}
|
||||
if len(f.calls) != 0 {
|
||||
t.Errorf("%d commands ran", len(f.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAsksGitForTheSshRemote(t *testing.T) {
|
||||
m, f := fetchMachine(t, typeScriptModule(`{}`), nil)
|
||||
if _, err := m.fetch(context.Background(), fetchArgs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var all []string
|
||||
for _, c := range f.calls {
|
||||
all = append(all, c.name+" "+strings.Join(c.args, " "))
|
||||
}
|
||||
lines := strings.Join(all, "\n")
|
||||
for _, want := range []string{"remote add origin ssh://mesh-forge/novox/mesh-catalog.git", "fetch --no-tags --depth 1 origin " + commitSHA, "checkout -q --detach " + commitSHA} {
|
||||
if !strings.Contains(lines, want) {
|
||||
t.Errorf("git was not asked %q:\n%s", want, lines)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) {
|
||||
m, _, work := newMachine(t)
|
||||
old, fresh := filepath.Join(work, "aaaa"), filepath.Join(work, "bbbb")
|
||||
for _, d := range []string{old, fresh, filepath.Join(work, "Not-A-Workspace")} {
|
||||
os.MkdirAll(d, 0o755)
|
||||
}
|
||||
os.Chtimes(old, time.Now().Add(-25*time.Hour), time.Now().Add(-25*time.Hour))
|
||||
os.Chtimes(filepath.Join(work, "Not-A-Workspace"), time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
|
||||
m.CleanWorkspaces(time.Now())
|
||||
if _, err := os.Stat(old); err == nil {
|
||||
t.Errorf("the old workspace is kept")
|
||||
}
|
||||
if _, err := os.Stat(fresh); err != nil {
|
||||
t.Errorf("the fresh workspace was removed")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(work, "Not-A-Workspace")); err != nil {
|
||||
t.Errorf("a directory that is no workspace was removed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
package main
|
||||
|
||||
// The forge side of a build module (novox/hq ADR 0466 §2 and §4, issue 1484). This file is the same for every build
|
||||
// module: mesh-build-node takes a copy and changes the three constants below.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const (
|
||||
forgeModule = "mesh-build-node"
|
||||
forgeHome = "/var/lib/mesh-build-node"
|
||||
forgeModuleD = "../.." // the module's directory, where module.json and files/ are
|
||||
)
|
||||
|
||||
type forgeManifest struct {
|
||||
Requires []any `json:"requires"`
|
||||
State []any `json:"state"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
func readForgeManifest(t *testing.T) forgeManifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(forgeModuleD, "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m forgeManifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func forgeTemplate(t *testing.T, r map[string]any) string {
|
||||
t.Helper()
|
||||
name, _ := r["template"].(string)
|
||||
if name == "" {
|
||||
t.Fatalf("resource %v names no template", r["id"])
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(forgeModuleD, filepath.FromSlash(name)))
|
||||
if err != nil {
|
||||
t.Fatalf("resource %v names %s: %v", r["id"], name, err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func forgeResource(t *testing.T, m forgeManifest, id string) map[string]any {
|
||||
t.Helper()
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
if r["owner"] != forgeModule {
|
||||
t.Errorf("%s is owned by %v, not the module's own account", id, r["owner"])
|
||||
}
|
||||
if p, _ := r["path"].(string); !strings.HasPrefix(p, "${dir:") || strings.Contains(p, "account-home") {
|
||||
t.Errorf("%s is at %q: a path of the account's own home (${dir:...}), never the operator's", id, p)
|
||||
}
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("no resource %q", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
var forgeSample = map[string]string{
|
||||
"known_host": "[novox.internal]:222 ssh-ed25519 AAAAC3Nza",
|
||||
"host_key": "ssh-ed25519 AAAAC3Nza",
|
||||
"ssh_host": "novox.internal",
|
||||
"ssh_alias": "mesh-forge",
|
||||
"ssh_port": "222",
|
||||
"ssh_base": "ssh://git@novox.internal:222/",
|
||||
"https_base": "https://git.novox.be/",
|
||||
}
|
||||
|
||||
// forgeRender fills ${bound:forge-account:<key>} with the sample values, as the provisioner client does.
|
||||
func forgeRender(t *testing.T, s string) string {
|
||||
t.Helper()
|
||||
for k, v := range forgeSample {
|
||||
s = strings.ReplaceAll(s, "${bound:forge-account:"+k+"}", v)
|
||||
}
|
||||
if strings.Contains(s, "${") {
|
||||
t.Errorf("unfilled placeholder in %q", s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// 1484/AC1: given sample bound values, the ssh config has Host mesh-forge, IdentitiesOnly yes and the key under the
|
||||
// module's own home; known_hosts carries the host key; git's config rewrites https_base to ssh://mesh-forge/; no
|
||||
// ${ is left.
|
||||
func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
|
||||
cfg := forgeResource(t, m, "ssh-config")
|
||||
want := "Host mesh-forge\n HostName novox.internal\n Port 222\n HostKeyAlias mesh-forge\n User git\n IdentityFile ~/.ssh/mesh_ed25519\n IdentitiesOnly yes\n"
|
||||
if got := forgeRender(t, forgeTemplate(t, cfg)); got != want {
|
||||
t.Errorf("ssh config = %q, want %q", got, want)
|
||||
}
|
||||
if cfg["path"] != "${dir:ssh-dir}/config" || cfg["mode"] != "0600" || cfg["into"] != nil {
|
||||
t.Errorf("ssh config resource = %v, want the whole file ${dir:ssh-dir}/config at mode 0600", cfg)
|
||||
}
|
||||
|
||||
kh := forgeResource(t, m, "known-hosts")
|
||||
if got := strings.TrimSpace(forgeRender(t, forgeTemplate(t, kh))); got != "mesh-forge ssh-ed25519 AAAAC3Nza" {
|
||||
t.Errorf("known_hosts = %q", got)
|
||||
}
|
||||
|
||||
g := forgeResource(t, m, "git-config")
|
||||
wantGit := "[url \"ssh://mesh-forge/\"]\n\tinsteadOf = https://git.novox.be/\n"
|
||||
if got := forgeRender(t, forgeTemplate(t, g)); got != wantGit {
|
||||
t.Errorf("git config = %q, want %q", got, wantGit)
|
||||
}
|
||||
|
||||
dir := forgeResource(t, m, "ssh-dir")
|
||||
if dir["mode"] != "0700" || dir["path"] != "${dir:state}/.ssh" {
|
||||
t.Errorf("ssh-dir = %v, want ${dir:state}/.ssh at mode 0700", dir)
|
||||
}
|
||||
}
|
||||
|
||||
// 1484/AC2: the module requires forge-account, keeps the state keys, and every file resource takes its text from a
|
||||
// template in the files/ archive (novox/hq ADR 0349 §2).
|
||||
func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
if !slices.Contains(m.Requires, any("forge-account")) {
|
||||
t.Errorf("requires = %v, want forge-account among them", m.Requires)
|
||||
}
|
||||
keeps := false
|
||||
for _, s := range m.State {
|
||||
if s == "keys" {
|
||||
keeps = true
|
||||
}
|
||||
}
|
||||
if !keeps {
|
||||
t.Errorf("state = %v lacks keys", m.State)
|
||||
}
|
||||
files := 0
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline {
|
||||
t.Errorf("resource %v keeps its text inline", r["id"])
|
||||
}
|
||||
if r["type"] == "file" {
|
||||
files++
|
||||
forgeTemplate(t, r)
|
||||
}
|
||||
}
|
||||
if files == 0 {
|
||||
t.Errorf("no file resource: the forge's ssh files are missing")
|
||||
}
|
||||
packs := 0
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a["kind"] == "archive" && a["from"] == "files" && a["name"] == "files" {
|
||||
packs++
|
||||
}
|
||||
}
|
||||
if packs != 1 {
|
||||
t.Errorf("the build packs files/ into %d archives named files, not one", packs)
|
||||
}
|
||||
}
|
||||
|
||||
// forgeEd25519Line is a public key line with a well-formed ed25519 blob.
|
||||
func forgeEd25519Line(comment string) string {
|
||||
blob := []byte{}
|
||||
for _, part := range [][]byte{[]byte("ssh-ed25519"), make([]byte, 32)} {
|
||||
l := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(l, uint32(len(part)))
|
||||
blob = append(append(blob, l...), part...)
|
||||
}
|
||||
return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(blob) + " " + comment
|
||||
}
|
||||
|
||||
type forgeKeys struct{ puts map[string][]byte }
|
||||
|
||||
func (f *forgeKeys) Put(key string, value any) (uint64, error) {
|
||||
raw, _ := json.Marshal(value)
|
||||
if f.puts == nil {
|
||||
f.puts = map[string][]byte{}
|
||||
}
|
||||
f.puts[key] = raw
|
||||
return uint64(len(f.puts)), nil
|
||||
}
|
||||
|
||||
// forgeKeygen stands in for ssh-keygen: it writes the pair the command names.
|
||||
func forgeKeygen(_ context.Context, _ []string, name string, args ...string) forgekey.Ran {
|
||||
var file, comment string
|
||||
for i, a := range args {
|
||||
switch a {
|
||||
case "-f":
|
||||
file = args[i+1]
|
||||
case "-C":
|
||||
comment = args[i+1]
|
||||
}
|
||||
}
|
||||
if name != "ssh-keygen" || file == "" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "unexpected"}
|
||||
}
|
||||
os.WriteFile(file, []byte("PRIVATE\n"), 0o600)
|
||||
os.WriteFile(file+".pub", []byte(forgeEd25519Line(comment)+"\n"), 0o644)
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
|
||||
// 1484/AC3: given the bundle started with an empty home, the keeper writes <node>.<module> with kind builder, and
|
||||
// the key is in the home's .ssh.
|
||||
func TestTheKeeperWritesTheModulesEntryWithKindBuilder(t *testing.T) {
|
||||
home, st := t.TempDir(), &forgeKeys{}
|
||||
if err := keeperFor(home, "ace", forgeModule, forgeKeygen, st).Once(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(home, ".ssh", "mesh_ed25519")); err != nil {
|
||||
t.Errorf("no key in an empty home: %v", err)
|
||||
}
|
||||
raw, ok := st.puts["ace."+forgeModule]
|
||||
if !ok || len(st.puts) != 1 {
|
||||
t.Fatalf("entries = %v, want one at ace.%s", st.puts, forgeModule)
|
||||
}
|
||||
var e forgekey.KeyEntry
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e.Kind != "builder" || e.Node != "ace" || e.Account != forgeModule || !strings.HasPrefix(e.Public, "ssh-ed25519 ") {
|
||||
t.Errorf("entry = %s", raw)
|
||||
}
|
||||
if strings.Contains(strings.ToUpper(string(raw)), "PRIVATE") {
|
||||
t.Errorf("the entry carries private material: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// The home the manifest places is the one the account is given, so the keeper's home and the templates' agree.
|
||||
func TestTheAccountsHomeIsTheOneTheManifestNames(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "user" && r["name"] == forgeModule && r["home"] != forgeHome {
|
||||
t.Errorf("the account's home is %v, want %s", r["home"], forgeHome)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// A fake machine for the step tests: a toolchain directory laid out as `npm ci --no-bin-links` leaves it, a work
|
||||
// directory, a clock that only the fake commands advance, and a runner that records every command and answers by
|
||||
// what the test set.
|
||||
|
||||
type call struct {
|
||||
dir string // env -C <dir>
|
||||
name string // the program after env -C <dir>
|
||||
args []string // its arguments
|
||||
env []string
|
||||
}
|
||||
|
||||
type fakeMachine struct {
|
||||
mu sync.Mutex
|
||||
calls []call
|
||||
now time.Time
|
||||
// cost is how long a program takes, by the base name of its path; answer is what it prints and its status;
|
||||
// effect runs the program's effects on the disk.
|
||||
cost map[string]time.Duration
|
||||
answer map[string]forgekey.Ran
|
||||
effect func(c call)
|
||||
}
|
||||
|
||||
func (f *fakeMachine) run(_ context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
c := call{name: name, args: args, env: env}
|
||||
key := filepath.Base(name)
|
||||
if name == "env" && len(args) >= 3 && args[0] == "-C" {
|
||||
c = call{dir: args[1], name: args[2], args: args[3:], env: env}
|
||||
key = filepath.Base(c.name)
|
||||
if key == "node" && len(c.args) > 0 {
|
||||
key = filepath.Base(c.args[0]) // node <tsc>
|
||||
}
|
||||
} else if name == "git" {
|
||||
for _, a := range args {
|
||||
if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files"}, a) {
|
||||
key = "git " + a
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
f.calls = append(f.calls, c)
|
||||
f.now = f.now.Add(f.cost[key])
|
||||
if f.effect != nil {
|
||||
f.effect(c)
|
||||
}
|
||||
return f.answer[key]
|
||||
}
|
||||
|
||||
func (f *fakeMachine) clock() time.Time { f.mu.Lock(); defer f.mu.Unlock(); return f.now }
|
||||
|
||||
// keys of the programs: tsc, esbuild, npm.
|
||||
|
||||
func newMachine(t *testing.T) (*Module, *fakeMachine, string) {
|
||||
t.Helper()
|
||||
home, cache, work := t.TempDir(), t.TempDir(), t.TempDir()
|
||||
f := &fakeMachine{now: time.Unix(1_000_000, 0), cost: map[string]time.Duration{}, answer: map[string]forgekey.Ran{}}
|
||||
m := &Module{Node: "ace", Recipe: Recipe{Steps: "1", Node: "v26.10.0", Lock: "lock"},
|
||||
Toolchain: &Toolchain{Paths: Paths{Home: home, Cache: cache}, Registry: func() (Registry, error) {
|
||||
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
|
||||
}},
|
||||
Work: work, Run: f.run, Clock: f.clock, Ctx: context.Background()}
|
||||
// The toolchain directory as npm ci leaves it, with no bin links.
|
||||
nm := filepath.Join(m.toolchainDir(), "node_modules")
|
||||
for _, p := range []string{"typescript/bin/tsc", "esbuild/bin/esbuild", "@esbuild/linux-x64/bin/esbuild",
|
||||
"@novox/mesh-sdk/dist/stdio.js", "@types/node/index.d.ts", "nats/lib/nats.js"} {
|
||||
write(t, filepath.Join(nm, p), "x", 0o644)
|
||||
}
|
||||
return m, f, work
|
||||
}
|
||||
|
||||
func write(t *testing.T, path, body string, mode os.FileMode) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// workspaceWith makes workspace "w1" of the machine with the files given (path under tree -> body).
|
||||
func workspaceWith(t *testing.T, work string, files map[string]string) string {
|
||||
t.Helper()
|
||||
for p, body := range files {
|
||||
write(t, filepath.Join(work, "w1", "tree", filepath.FromSlash(p)), body, 0o644)
|
||||
}
|
||||
return "w1"
|
||||
}
|
||||
|
||||
func symlinksUnder(t *testing.T, root string) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err == nil && info.Mode()&os.ModeSymlink != 0 {
|
||||
out = append(out, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func joined(c call) string { return c.name + " " + strings.Join(c.args, " ") }
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
// The module's own forge key (novox/hq ADR 0466 §2, issue 1484): mesh-sdk's key keeper, kind builder, for the
|
||||
// module's account in its home. It makes ~/.ssh/mesh_ed25519 when it is missing and writes one entry to this
|
||||
// module's `keys` state at <node>.<account>, which gitea's provider reads to register the key on the forge user
|
||||
// that reads and pushes nothing (ADR 0466 §3).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// keeperFor is this module's keeper: go/forgekey's, of kind builder, for account in home on node, whose entry lies
|
||||
// at <node>.<account> of the keys state.
|
||||
func keeperFor(home, node, account string, run forgekey.Runner, state forgekey.KeyPutter) *forgekey.KeyKeeper {
|
||||
return &forgekey.KeyKeeper{Home: home, Node: node, Account: account, Kind: forgekey.KindBuilder, Run: run, State: state, Now: time.Now}
|
||||
}
|
||||
|
||||
// startKeeper runs the keeper when the runtime launched this process for a node: outside a launch — a build, a
|
||||
// check — there is no state to put the entry in.
|
||||
func startKeeper(ctx context.Context) {
|
||||
node := strings.TrimSpace(os.Getenv("MESH_NODE"))
|
||||
u, err := user.Current()
|
||||
if node == "" || err != nil || u.HomeDir == "" {
|
||||
return
|
||||
}
|
||||
k := keeperFor(u.HomeDir, node, u.Username, forgekey.ExecRunner, keysState{})
|
||||
k.Keep(ctx, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-node] "+f+"\n", a...) })
|
||||
}
|
||||
|
||||
type keysState struct{}
|
||||
|
||||
func (keysState) Put(key string, value any) (uint64, error) {
|
||||
return stdio.State("keys").Put(key, value)
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// mesh-build-node: the Node and TypeScript build module (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470; design
|
||||
// 47 R4c-3, issue 1488). A Go tools bundle the node's runtime launches as the module's own account (runs-as). It names
|
||||
// the one Node version it builds with, installs the TypeScript toolchain its own lock names, writes the Node version
|
||||
// it finds to <node>.typescript of its `toolchains` state at start and before every step, and refuses every step on
|
||||
// a node that has another: its health is then unhealthy, so the controller raises the node's condition. What it
|
||||
// says, it says on stderr; stdout is the MCP channel.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// CacheDir is the cache directory, outside the account's home (ADR 0466 §1; module.json resource `cache`).
|
||||
const CacheDir = "/var/lib/mesh-build-node-cache"
|
||||
|
||||
// WorkDir is the work directory, outside the home (module.json resource `work`).
|
||||
const WorkDir = "/var/lib/mesh-build-node-work"
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-node] "+format+"\n", a...) }
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
home := ""
|
||||
if u, err := user.Current(); err == nil {
|
||||
home = u.HomeDir
|
||||
}
|
||||
paths := Paths{Home: home, Cache: CacheDir}
|
||||
registry := func() (Registry, error) {
|
||||
return ReadRegistry(filepath.Join(home, "package-registry.json"), filepath.Join(home, "package-registry.secret"))
|
||||
}
|
||||
m := &Module{
|
||||
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
|
||||
Reload: func() (Recipe, error) { return ReadRecipe(paths.LockPath()) },
|
||||
Probe: ProbeNode,
|
||||
Toolchain: &Toolchain{Paths: paths, Run: forgekey.ExecRunnerClean(0), Registry: registry},
|
||||
Toolchains: stdio.State(delivery.ToolchainsState).Put,
|
||||
Emit: func(t string, b any) error { return stdio.Emit(t, b) },
|
||||
Steps: stdio.State(delivery.StepsState).Put,
|
||||
Work: WorkDir,
|
||||
Run: forgekey.ExecRunnerClean(0),
|
||||
Ctx: ctx,
|
||||
Say: logf,
|
||||
}
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
}
|
||||
go startKeeper(ctx)
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The catalogue's root and this module's directory.
|
||||
const (
|
||||
catalogueRoot = "../../../.."
|
||||
moduleDir = "../.."
|
||||
)
|
||||
|
||||
func readManifest(t *testing.T) catalogue.Manifest {
|
||||
t.Helper()
|
||||
m, err := catalogue.ReadTreeManifest(catalogueRoot, moduleDir)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's validator refuses module.json: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func resource(t *testing.T, m catalogue.Manifest, id string) map[string]any {
|
||||
t.Helper()
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("module.json declares no resource %q", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1488/AC1: mesh-sdk's validator accepts the module (alone on a shelf too), and it runs as mesh-build-node with its
|
||||
// home at mode 0700, requires user-namespaces and declares the four packages and carries toolchain/package-lock.json.
|
||||
func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if problems := catalogue.CatalogueProblems(catalogue.Shelf{m.Module: m}); len(problems) > 0 {
|
||||
t.Errorf("the catalogue's rules refuse it: %v", problems)
|
||||
}
|
||||
if m.Module != "mesh-build-node" || m.RunsAs != "mesh-build-node" {
|
||||
t.Errorf("module %q runs as %q, want mesh-build-node as itself", m.Module, m.RunsAs)
|
||||
}
|
||||
account := resource(t, m, "account")
|
||||
if account["type"] != "user" || account["name"] != "mesh-build-node" || account["shell"] != "/usr/bin/nologin" || account["home"] != "/var/lib/mesh-build-node" {
|
||||
t.Errorf("account = %v", account)
|
||||
}
|
||||
home := resource(t, m, "state")
|
||||
if home["type"] != "directory" || home["mode"] != "0700" || home["owner"] != "mesh-build-node" || home["place"] != "." {
|
||||
t.Errorf("home = %v, want its directory at mode 0700 owned by the account", home)
|
||||
}
|
||||
if !slices.Contains(m.Capabilities, "user-namespaces") {
|
||||
t.Errorf("capabilities = %v, want user-namespaces", m.Capabilities)
|
||||
}
|
||||
packages := []string{}
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "package" {
|
||||
packages = append(packages, r["package"].(string))
|
||||
}
|
||||
}
|
||||
slices.Sort(packages)
|
||||
if want := []string{"git", "nodejs", "npm", "openssh"}; !slices.Equal(packages, want) {
|
||||
t.Errorf("packages = %v, want %v", packages, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The work and cache directories are outside the home (ADR 0466 §1), owned by the account.
|
||||
func TestWorkAndCacheDirectoriesLieOutsideTheHome(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
for _, id := range []string{"work", "cache"} {
|
||||
r := resource(t, m, id)
|
||||
path, _ := r["path"].(string)
|
||||
if path == "" || path == "/var/lib/mesh-build-node" || len(path) > len("/var/lib/mesh-build-node") && path[:len("/var/lib/mesh-build-node/")] == "/var/lib/mesh-build-node/" {
|
||||
t.Errorf("%s is at %q, inside or at the home", id, path)
|
||||
}
|
||||
if r["owner"] != "mesh-build-node" {
|
||||
t.Errorf("%s is owned by %v", id, r["owner"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The states are the three the module keeps, no seat is claimed (ADR 0469 §6), and the health is the module's own
|
||||
// tool, which the manifest also lists among its tools.
|
||||
func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
states := []string{}
|
||||
for _, s := range m.State {
|
||||
states = append(states, s.Name)
|
||||
}
|
||||
slices.Sort(states)
|
||||
if want := []string{"keys", "steps", "toolchains"}; !slices.Equal(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if len(m.Claims) != 0 {
|
||||
t.Errorf("claims = %v, want no seat", m.Claims)
|
||||
}
|
||||
if !slices.Contains(m.Tools, HealthTool) {
|
||||
t.Errorf("tools = %v lack %s", m.Tools, HealthTool)
|
||||
}
|
||||
raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json"))
|
||||
if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) {
|
||||
t.Errorf("runtime-health does not name %s", HealthTool)
|
||||
}
|
||||
}
|
||||
|
||||
// Every tool the manifest names is served, and every tool served is named.
|
||||
func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
served := []string{}
|
||||
for _, tool := range (&Module{}).Tools((&Module{}).StepTools()) {
|
||||
served = append(served, tool.Name)
|
||||
}
|
||||
slices.Sort(served)
|
||||
named := slices.Clone(m.Tools)
|
||||
slices.Sort(named)
|
||||
if !slices.Equal(served, named) {
|
||||
t.Errorf("served %v, manifest names %v", served, named)
|
||||
}
|
||||
}
|
||||
|
||||
// 1488/AC1: the module carries the committed lock beside its package.json, and the manifest unpacks them into the
|
||||
// home's toolchain directory, where the program reads them.
|
||||
func TestTheModuleCarriesItsToolchainLockAndTheManifestPlacesIt(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
for _, name := range []string{"package.json", "package-lock.json"} {
|
||||
if _, err := os.Stat(filepath.Join(moduleDir, "toolchain", name)); err != nil {
|
||||
t.Errorf("the module carries no toolchain/%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(m.Requires, "npm-package-registry") {
|
||||
t.Errorf("requires = %v lack npm-package-registry, which the install resolves @novox from", m.Requires)
|
||||
}
|
||||
}
|
||||
|
||||
// 1488/AC1: the lock names the versions of the recipe: typescript 5.9.3, esbuild 0.25.12, @novox/mesh-sdk 0.1.14
|
||||
// and nats, all by the lock's own hash (read from the file the module carries).
|
||||
func TestTheLockNamesTheToolchainVersions(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, "toolchain", "package-lock.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var lock struct {
|
||||
Packages map[string]struct {
|
||||
Version string `json:"version"`
|
||||
} `json:"packages"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &lock); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for pkg, want := range map[string]string{"typescript": "5.9.3", "esbuild": "0.25.12", "@novox/mesh-sdk": "0.1.14", "nats": "2.29.3", "@types/node": "22.20.1"} {
|
||||
if got := lock.Packages["node_modules/"+pkg].Version; got != want {
|
||||
t.Errorf("%s = %q in the lock, want %s", pkg, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1488 (the catalogue's manifest check): a manifest names no host, the mesh being name-agnostic. The tree-read
|
||||
// manifest carries each template's text as the resource's content, and the toolchain's lock names registries, so it is
|
||||
// shipped as an archive artifact instead, never as a template. No resource of the module, read as the mesh reads it,
|
||||
// carries content naming a host, and the raw module.json names none either.
|
||||
func TestNoResourceCarriesContentNamingAHost(t *testing.T) {
|
||||
host := regexp.MustCompile(`(?i)://[a-z0-9]|\b[a-z0-9-]+(\.[a-z0-9-]+)*\.(com|org|net|be|io|dev)\b`)
|
||||
m := readManifest(t)
|
||||
for i, r := range m.Resources {
|
||||
if c, _ := r["content"].(string); host.MatchString(c) {
|
||||
t.Errorf("resources[%d] (%v) carries content naming %q", i, r["id"], host.FindString(c))
|
||||
}
|
||||
}
|
||||
raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json"))
|
||||
if h := host.FindString(string(raw)); h != "" {
|
||||
t.Errorf("module.json names %q", h)
|
||||
}
|
||||
}
|
||||
|
||||
// 1488: the toolchain's package.json and lock are shipped in the archive artifact `toolchain` and unpacked into the
|
||||
// home's toolchain directory, where the program reads them.
|
||||
func TestTheToolchainFilesAreAnArchiveUnpackedIntoTheHome(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
r := resource(t, m, "toolchain-dir")
|
||||
if r["type"] != "archive" || r["path"] != "${dir:state}/toolchain" || r["owner"] != "mesh-build-node" {
|
||||
t.Errorf("toolchain-dir = %v, want an archive unpacked at ${dir:state}/toolchain", r)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(moduleDir, "toolchain", "package-lock.json")); err != nil {
|
||||
t.Errorf("the module carries no toolchain/package-lock.json: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// CheckRound is how often the module reads the installed toolchain again, beside the check before every step and
|
||||
// the one every health look makes.
|
||||
const CheckRound = 10 * time.Minute
|
||||
|
||||
// Module is the Node build module's one running state: the node it runs on, its recipe, the toolchain guard, the
|
||||
// toolchain directory and the states it writes (toolchains, steps).
|
||||
type Module struct {
|
||||
Node string
|
||||
// Recipe is the recipe in force. Reload, when set, reads it again at every check (the lock file the mesh
|
||||
// writes can change under a running module); a failure to read it is a refusal.
|
||||
Recipe Recipe
|
||||
Reload func() (Recipe, error)
|
||||
// Probe reads the installed Node version (ProbeNode).
|
||||
Probe func(ctx context.Context) (string, error)
|
||||
// Toolchain installs the toolchain directory; nil in a module that only guards.
|
||||
Toolchain *Toolchain
|
||||
// Work is the work directory, outside the home: a workspace is <Work>/<step id>/tree. Run runs the commands of a
|
||||
// step (a fake in a test). Clock is the time the steps are timed by.
|
||||
Work string
|
||||
Run forgekey.Runner
|
||||
Clock func() time.Time
|
||||
// Toolchains and Steps are the Put of the states of those names.
|
||||
Toolchains, Steps buildstep.Put
|
||||
// Emit tells the bus an event (stdio.Emit); nil emits nothing. The guard emits toolchain.reported after each write of
|
||||
// the toolchains key, so the indexer, which holds no timer, learns that the bucket exists (issue 1444).
|
||||
Emit func(eventType string, body any) error
|
||||
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
|
||||
Ctx context.Context
|
||||
Say func(format string, a ...any)
|
||||
|
||||
mu sync.Mutex // guards Recipe
|
||||
install sync.Mutex // one install at a time
|
||||
}
|
||||
|
||||
func (m *Module) recipe() Recipe {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.Recipe
|
||||
}
|
||||
|
||||
func (m *Module) guard(r Recipe) buildstep.Guard {
|
||||
return buildstep.Guard{Language: "typescript", Want: r.Node, Recipe: r.Version(), Probe: m.Probe, Node: m.Node, Put: m.reportingPut()}
|
||||
}
|
||||
|
||||
// Check reads the installed toolchain, writes <node>.typescript of the toolchains state and answers nil, or the
|
||||
// refusal (buildstep.Guard.Check). It runs at start, before every step, at every health look and every CheckRound.
|
||||
func (m *Module) Check(ctx context.Context) error {
|
||||
if m.Reload != nil {
|
||||
r, err := m.Reload()
|
||||
if err != nil {
|
||||
return &buildstep.Refusal{Text: err.Error(), Want: RecipeNode}
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.Recipe = r
|
||||
m.mu.Unlock()
|
||||
}
|
||||
return m.guard(m.recipe()).Check(ctx)
|
||||
}
|
||||
|
||||
// Prepare is Check and then the toolchain directory of the recipe in force: what every step runs behind.
|
||||
func (m *Module) Prepare(ctx context.Context) error {
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Toolchain == nil {
|
||||
return nil
|
||||
}
|
||||
m.install.Lock()
|
||||
defer m.install.Unlock()
|
||||
return m.Toolchain.Ensure(ctx, m.recipe().Version())
|
||||
}
|
||||
|
||||
// Start prepares once, as the module starts, and then every CheckRound until ctx ends.
|
||||
func (m *Module) Start(ctx context.Context) {
|
||||
for {
|
||||
if err := m.Prepare(ctx); err != nil && m.Say != nil {
|
||||
m.Say("the toolchain: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(CheckRound):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy while the toolchain installed is not
|
||||
// the recipe's, saying both versions, so the controller raises the node's condition. It looks again each time it is
|
||||
// asked.
|
||||
func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
r := m.recipe()
|
||||
return health.Healthy(fmt.Sprintf("node %s installed, as the recipe %s builds with", r.Node, r.Version()))
|
||||
}
|
||||
|
||||
// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on
|
||||
// its key in the steps state. The guard runs inside the step, before run, so a node that refuses ends every step
|
||||
// refused with both versions named.
|
||||
func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) {
|
||||
if err := m.Prepare(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return run(ctx, args)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, _ := delivery.StepKey(m.Node, id)
|
||||
return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error.
|
||||
func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
if err := m.Prepare(m.Ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return run(m.Ctx, args)
|
||||
}
|
||||
}
|
||||
|
||||
// Status is what the module reports of itself: the node, the recipe, the toolchain found and why it refuses, if it
|
||||
// does.
|
||||
func (m *Module) Status(ctx context.Context) map[string]any {
|
||||
err := m.Check(ctx)
|
||||
installed, _ := m.Probe(ctx)
|
||||
r := m.recipe()
|
||||
out := map[string]any{"node": m.Node, "recipe": r.Version(), "node-version": r.Node, "installed": installed, "refusing": err != nil}
|
||||
if m.Toolchain != nil {
|
||||
out["toolchain-dir"] = m.Toolchain.Paths.Dir(r.Version())
|
||||
out["toolchain-installed"] = m.Toolchain.Present(r.Version())
|
||||
}
|
||||
if err != nil {
|
||||
out["why"] = err.Error()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Module) now() time.Time {
|
||||
if m.Clock != nil {
|
||||
return m.Clock()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// defaultPath is where a step looks for node, npm, git and ssh when this process was given no PATH.
|
||||
const defaultPath = "/usr/local/bin:/usr/bin:/bin"
|
||||
|
||||
// baseEnv is the environment of a step's command beyond what the step adds: PATH, the account's own HOME (its ssh
|
||||
// files and git configuration) and a fixed locale. A command is given these and its own words, never this process's
|
||||
// environment, which holds the module's bus credentials (ADR 0466 §4).
|
||||
func (m *Module) baseEnv() []string {
|
||||
return []string{"PATH=" + pathOrDefault(), "HOME=" + m.Toolchain.homeOrEmpty(), "LC_ALL=C"}
|
||||
}
|
||||
|
||||
func pathOrDefault() string {
|
||||
if path := os.Getenv("PATH"); path != "" {
|
||||
return path
|
||||
}
|
||||
return defaultPath
|
||||
}
|
||||
|
||||
// ToolchainReported is the event the guard emits after it writes the toolchains key.
|
||||
const ToolchainReported = "toolchain.reported"
|
||||
|
||||
// reportingPut is the toolchains Put that then emits toolchain.reported with the node and the language. A write that
|
||||
// failed emits nothing; an emit that failed is said and does not refuse the step.
|
||||
func (m *Module) reportingPut() buildstep.Put {
|
||||
return func(key string, value any) (uint64, error) {
|
||||
rev, err := m.Toolchains(key, value)
|
||||
if err != nil || m.Emit == nil {
|
||||
return rev, err
|
||||
}
|
||||
if eerr := m.Emit(ToolchainReported, map[string]string{"node": m.Node, "language": "typescript"}); eerr != nil && m.Say != nil {
|
||||
m.Say("the event %s could not be emitted: %v", ToolchainReported, eerr)
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// keptStates stands in for the toolchains and the steps states.
|
||||
type keptStates struct {
|
||||
mu sync.Mutex
|
||||
toolchains map[string]delivery.Toolchain
|
||||
steps map[string][]delivery.Step
|
||||
ended chan string
|
||||
}
|
||||
|
||||
func newStates() *keptStates {
|
||||
return &keptStates{toolchains: map[string]delivery.Toolchain{}, steps: map[string][]delivery.Step{}, ended: make(chan string, 16)}
|
||||
}
|
||||
|
||||
func (s *keptStates) putToolchain(key string, value any) (uint64, error) {
|
||||
tc, err := delivery.DecodeToolchain(value.(json.RawMessage))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.toolchains[key] = tc
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (s *keptStates) putStep(key string, value any) (uint64, error) {
|
||||
st, err := delivery.DecodeStep(value.(json.RawMessage))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.steps[key] = append(s.steps[key], st)
|
||||
s.mu.Unlock()
|
||||
if st.State != delivery.StepRunning {
|
||||
s.ended <- key
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (s *keptStates) toolchain(key string) (delivery.Toolchain, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
tc, ok := s.toolchains[key]
|
||||
return tc, ok
|
||||
}
|
||||
|
||||
func (s *keptStates) waitEnd(t *testing.T) delivery.Step {
|
||||
t.Helper()
|
||||
select {
|
||||
case key := <-s.ended:
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v := s.steps[key]
|
||||
return v[len(v)-1]
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the step never ended on its key")
|
||||
return delivery.Step{}
|
||||
}
|
||||
}
|
||||
|
||||
func moduleWith(states *keptStates, installed string) *Module {
|
||||
return &Module{Node: "ace", Recipe: Recipe{Steps: "1", Node: "v26.10.0", Lock: "lock"},
|
||||
Probe: func(context.Context) (string, error) { return installed, nil },
|
||||
Toolchains: states.putToolchain, Steps: states.putStep, Ctx: context.Background()}
|
||||
}
|
||||
|
||||
// 1488/AC3: a probe answering v26.10.0: the bundle's start writes ace.typescript with that version and the recipe's,
|
||||
// and the health tool answers healthy.
|
||||
func TestStartWritesTheToolchainAndHealthIsHealthy(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "v26.10.0")
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go m.Start(ctx)
|
||||
var tc delivery.Toolchain
|
||||
for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
|
||||
if got, ok := states.toolchain("ace.typescript"); ok {
|
||||
tc = got
|
||||
break
|
||||
}
|
||||
}
|
||||
if tc.Version != "v26.10.0" || tc.Recipe != m.Recipe.Version() || tc.Language != "typescript" {
|
||||
t.Fatalf("ace.typescript = %+v, want v26.10.0 and recipe %s", tc, m.Recipe.Version())
|
||||
}
|
||||
if a := m.Health(ctx); a.State != health.StateHealthy || !a.Healthy {
|
||||
t.Errorf("health = %+v, want healthy", a)
|
||||
}
|
||||
}
|
||||
|
||||
// 1488/AC3: a probe answering v26.11.0: every step tool's step ends refused naming both versions, ace.typescript
|
||||
// holds v26.11.0, and the health tool answers unhealthy naming both.
|
||||
func TestAnotherNodeRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "v26.11.0")
|
||||
ran := false
|
||||
run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
|
||||
steps := append([]StepTool{
|
||||
{Name: "long_probe", Long: true, Run: run},
|
||||
{Name: "short_probe", Run: run},
|
||||
}, (&Module{}).StepTools()...)
|
||||
for _, tool := range m.Tools(steps) {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool {
|
||||
continue
|
||||
}
|
||||
var reason string
|
||||
out, err := tool.Run(map[string]any{})
|
||||
if isLong(steps, tool.Name) {
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tool.Name, err)
|
||||
}
|
||||
if out.(map[string]any)["step"] == "" {
|
||||
t.Fatalf("%s answered no step id: %v", tool.Name, out)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepRefused {
|
||||
t.Errorf("%s: the step ended %s, want refused", tool.Name, st.State)
|
||||
}
|
||||
reason = st.Reason
|
||||
} else {
|
||||
if err == nil {
|
||||
t.Fatalf("%s: answered %v, want a refusal", tool.Name, out)
|
||||
}
|
||||
reason = err.Error()
|
||||
}
|
||||
if !strings.Contains(reason, "v26.11.0") || !strings.Contains(reason, "v26.10.0") {
|
||||
t.Errorf("%s: the refusal %q does not name both versions", tool.Name, reason)
|
||||
}
|
||||
}
|
||||
if ran {
|
||||
t.Errorf("a step ran on a node that refuses")
|
||||
}
|
||||
if tc, _ := states.toolchain("ace.typescript"); tc.Version != "v26.11.0" {
|
||||
t.Errorf("ace.typescript holds %q, want the installed v26.11.0", tc.Version)
|
||||
}
|
||||
a := m.Health(context.Background())
|
||||
if a.State != health.StateUnhealthy || !strings.Contains(a.Why, "v26.11.0") || !strings.Contains(a.Why, "v26.10.0") {
|
||||
t.Errorf("health = %+v, want unhealthy naming both versions", a)
|
||||
}
|
||||
}
|
||||
|
||||
func isLong(steps []StepTool, name string) bool {
|
||||
for _, s := range steps {
|
||||
if s.Name == name {
|
||||
return s.Long
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// 1488/AC2: the recipe version moves with one byte of the lock and with the Node version, and with the recipe's own
|
||||
// version; equal recipes have the same one.
|
||||
func TestRecipeVersionMovesWithEachOfItsValues(t *testing.T) {
|
||||
lock := []byte(`{"lockfileVersion":3}`)
|
||||
base := Recipe{Steps: "1", Node: "v26.10.0", Lock: LockHash(lock)}
|
||||
if base.Version() != (Recipe{Steps: "1", Node: "v26.10.0", Lock: LockHash(lock)}).Version() {
|
||||
t.Errorf("one recipe has two versions")
|
||||
}
|
||||
seen := map[string]string{base.Version(): "base"}
|
||||
for name, r := range map[string]Recipe{
|
||||
"one byte of the lock": {Steps: "1", Node: "v26.10.0", Lock: LockHash([]byte(`{"lockfileVersion":4}`))},
|
||||
"node": {Steps: "1", Node: "v26.11.0", Lock: LockHash(lock)},
|
||||
"steps": {Steps: "2", Node: "v26.10.0", Lock: LockHash(lock)},
|
||||
} {
|
||||
v := r.Version()
|
||||
if was, dup := seen[v]; dup {
|
||||
t.Errorf("the recipe differing in %s has the version of the %s recipe: %s", name, was, v)
|
||||
}
|
||||
seen[v] = name
|
||||
}
|
||||
}
|
||||
|
||||
// 1488/AC2: the recipe reads the lock file's bytes: a lock changed on disk is another recipe version, a lock missing
|
||||
// is an error.
|
||||
func TestReadRecipeHashesTheLockFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "package-lock.json")
|
||||
if _, err := ReadRecipe(path); err == nil {
|
||||
t.Errorf("a missing lock was accepted")
|
||||
}
|
||||
os.WriteFile(path, []byte("a"), 0o600)
|
||||
a, err := ReadRecipe(path)
|
||||
if err != nil || a.Node != "v26.10.0" {
|
||||
t.Fatalf("= %+v, %v", a, err)
|
||||
}
|
||||
os.WriteFile(path, []byte("b"), 0o600)
|
||||
b, _ := ReadRecipe(path)
|
||||
if a.Version() == b.Version() {
|
||||
t.Errorf("one byte of the lock did not move the recipe version")
|
||||
}
|
||||
}
|
||||
|
||||
// 1488/AC3: a lock changed under a running module is read at the next check, and a lock that cannot be read refuses.
|
||||
func TestACheckReadsTheRecipeAgain(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "v26.10.0")
|
||||
r2 := Recipe{Steps: "1", Node: "v26.10.0", Lock: "other"}
|
||||
m.Reload = func() (Recipe, error) { return r2, nil }
|
||||
if err := m.Check(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc, _ := states.toolchain("ace.typescript"); tc.Recipe != r2.Version() {
|
||||
t.Errorf("ace.typescript holds recipe %s, want the reloaded %s", tc.Recipe, r2.Version())
|
||||
}
|
||||
m.Reload = func() (Recipe, error) { return Recipe{}, errors.New("no lock") }
|
||||
if err := m.Check(context.Background()); err == nil || m.Health(context.Background()).Healthy {
|
||||
t.Errorf("a lock that cannot be read neither refused nor made the module unhealthy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNodeVersion(t *testing.T) {
|
||||
if v, err := parseNodeVersion("v26.10.0\n"); err != nil || v != "v26.10.0" {
|
||||
t.Errorf("= %q, %v", v, err)
|
||||
}
|
||||
if _, err := parseNodeVersion("not node"); err == nil {
|
||||
t.Errorf("accepted a text that is not `node --version`'s")
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user