diff --git a/.gitignore b/.gitignore index ab0142b4..9a589bdf 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ modules/mesh-files/cmd/mesh-files/mesh-files # The provisioner client built in place (issue 671). modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-client modules/brother/cmd/brother/brother +modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go diff --git a/checks/owned/main.go b/checks/owned/main.go index 09512d5d..67762f06 100644 --- a/checks/owned/main.go +++ b/checks/owned/main.go @@ -160,8 +160,14 @@ func Check(root string) ([]Finding, error) { } dirs := map[string]string{} for _, r := range m.Resources { - if r.ID != "" && r.Path != "" { + switch { + case r.ID != "" && r.Path != "": dirs[r.ID] = r.Path + case r.ID != "" && r.Type == "directory": + // A directory the mesh places for the module (`place`): where it lies is the node-engine's, but it is + // the module's own, so a path under it belongs to this module and to no other's directory of the same + // id (issue 1484: two runs-as modules each keep a .ssh in their own state directory). + dirs[r.ID] = "/placed/" + m.Module + "/" + r.ID } } for _, r := range m.Resources { diff --git a/checks/owned/main_test.go b/checks/owned/main_test.go index 0882ba47..6426244f 100644 --- a/checks/owned/main_test.go +++ b/checks/owned/main_test.go @@ -130,3 +130,18 @@ func TestAnArchiveIsNotAContainerAndSameModuleIsFine(t *testing.T) { t.Fatalf("refused %+v", found) } } + +// Issue 1484: a directory the mesh places for a module (no path, `place`) is that module's own: another module's +// directory of the same id, and a path under it, are not inside it. +func TestPlacedDirectoriesOfTwoModulesAreTheirOwn(t *testing.T) { + root := t.TempDir() + for _, name := range []string{"first", "second"} { + manifest(t, root, name, mod(name, + `{"id":"state","type":"directory","place":".","unknown":"program"}`, + dir("ssh", "${dir:state}/.ssh", `,"unknown":"program"`), + `{"id":"conf","type":"file","path":"${dir:ssh}/config"}`)) + } + if found := check(t, root); len(found) != 0 { + t.Fatalf("refused %+v", found) + } +} diff --git a/checks/sshdir/main.go b/checks/sshdir/main.go index 0b7eb4ab..babf2c5c 100644 --- a/checks/sshdir/main.go +++ b/checks/sshdir/main.go @@ -5,7 +5,7 @@ // go run . // // Every string of a manifest is read, wherever it stands, except prose and a file's text (Prose): a path, a -// place, a volume. Exits 1 on a finding. +// place, a volume; a path inside a directory of the module's own (`${dir:}`) is not ~/.ssh. Exits 1 on a finding. package main import ( @@ -15,6 +15,7 @@ import ( "path/filepath" "regexp" "sort" + "strings" ) // SSHModule is the one module allowed to name a path under ~/.ssh. @@ -60,7 +61,7 @@ func Check(root string) ([]Finding, error) { if module == SSHModule { continue } - walk(m, "", func(key, value string) { + walk(m, "", dirsOf(m), func(key, value string) { out = append(out, Finding{Module: module, Key: key, Value: value}) }) } @@ -68,7 +69,7 @@ func Check(root string) ([]Finding, error) { } // walk calls found for every string under v, outside Prose, that names a path under ~/.ssh. -func walk(v any, key string, found func(key, value string)) { +func walk(v any, key string, dirs map[string]string, found func(key, value string)) { switch x := v.(type) { case map[string]any: keys := make([]string, 0, len(x)) @@ -78,20 +79,68 @@ func walk(v any, key string, found func(key, value string)) { sort.Strings(keys) for _, k := range keys { if !Prose[k] { - walk(x[k], k, found) + walk(x[k], k, dirs, found) } } case []any: for _, e := range x { - walk(e, key, found) + walk(e, key, dirs, found) } case string: - if underSSH.MatchString(x) { + // A path under a directory the mesh places for the module (${dir:} of a directory with a place and no path, + // as a runs-as module's home) is that account's own .ssh, not the operator's ~/.ssh that ssh-client sweeps + // (issue 1484, ADR 0466 §4). Any other ${dir:} is resolved to the path its resource declares and judged. + if resolved, placed := resolveDir(x, dirs); !placed && underSSH.MatchString(resolved) { found(key, x) } } } +var ofDir = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9_-]*)\}`) + +// placedDir marks a directory the mesh places for the module: it has no path of its own. +const placedDir = "\x00placed" + +// dirsOf are the module's directories by id: the path each declares, or placedDir for one with a place and no path. +func dirsOf(m any) map[string]string { + dirs := map[string]string{} + obj, _ := m.(map[string]any) + list, _ := obj["resources"].([]any) + for _, r := range list { + res, _ := r.(map[string]any) + id, _ := res["id"].(string) + path, _ := res["path"].(string) + switch { + case id == "": + case path != "": + dirs[id] = path + case res["type"] == "directory": + dirs[id] = placedDir + } + } + return dirs +} + +// resolveDir follows a leading ${dir:} through the module's directories. It answers the path with the references +// resolved, and whether it ends under a placed directory. +func resolveDir(x string, dirs map[string]string) (string, bool) { + for i := 0; i < 8; i++ { + m := ofDir.FindStringSubmatch(x) + if m == nil { + return x, false + } + to, ok := dirs[m[1]] + if !ok { + return x, false + } + if to == placedDir { + return x, true + } + x = to + strings.TrimPrefix(x, m[0]) + } + return x, false +} + func main() { root := "." if len(os.Args) > 1 { diff --git a/checks/sshdir/main_test.go b/checks/sshdir/main_test.go index ca78b3e7..a7df3332 100644 --- a/checks/sshdir/main_test.go +++ b/checks/sshdir/main_test.go @@ -37,6 +37,36 @@ func TestAnotherModuleNamingSSHIsRefused(t *testing.T) { } } +// Issue 1484: a runs-as module's own account has a .ssh of its own, inside its home (a directory of the module's); +// that is not the operator's ~/.ssh, which only ssh-client names. A path outside any ${dir:} is still refused. +func TestAModulesOwnDirectoryMayHoldADotSSH(t *testing.T) { + root := t.TempDir() + manifest(t, root, "builder", `{"module":"builder","resources":[{"id":"state","type":"directory","place":"."},{"id":"ssh-dir","type":"directory","path":"${dir:state}/.ssh"},{"path":"${dir:ssh-dir}/config"}]}`) + manifest(t, root, "other", `{"module":"other","resources":[{"path":"${machine:account-home}/.ssh/config"}]}`) + found, err := Check(root) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 || found[0].Module != "other" { + t.Fatalf("refused %+v; want only other", found) + } +} + +// Issue 1484 review: only a directory the mesh places is the module's own home. A ${dir:} that resolves to the +// operator's home is judged by the path it resolves to. +func TestADirResolvingToTheOperatorsHomeIsStillRefused(t *testing.T) { + root := t.TempDir() + manifest(t, root, "escape", `{"module":"escape","resources":[{"id":"h","type":"directory","path":"${machine:account-home}","unknown":"module"},{"path":"${dir:h}/.ssh/config"}]}`) + manifest(t, root, "placed", `{"module":"placed","resources":[{"id":"state","type":"directory","place":".","unknown":"program"},{"id":"s","type":"directory","path":"${dir:state}/.ssh"},{"path":"${dir:s}/config"}]}`) + found, err := Check(root) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 || found[0].Module != "escape" { + t.Fatalf("refused %+v; want only escape", found) + } +} + func TestTheCatalogueHasNone(t *testing.T) { found, err := Check("../..") if err != nil { diff --git a/modules/gitea/cmd/forge-account-provider/builder_test.go b/modules/gitea/cmd/forge-account-provider/builder_test.go new file mode 100644 index 00000000..9bfd0408 --- /dev/null +++ b/modules/gitea/cmd/forge-account-provider/builder_test.go @@ -0,0 +1,356 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/provider" + + "gitea/internal/forgehttp" +) + +// Issue 1481 (novox/hq ADR 0466 §3): the build modules' keys on one forge user that reads and pushes nothing. The forge +// is faked with the teams, members and keys it keeps, so a team's shape and a key's right can be read back. + +type forgeKey struct { + ID int64 + Title string + Key string + ReadOnly bool +} + +type teamForge struct { + mu sync.Mutex + users map[string]bool + teams map[string]map[string]any // by name: the shape last sent + ids map[string]int64 + members map[string][]string // by team name + keys map[string][]forgeKey + next int64 +} + +func newTeamForge() *teamForge { + return &teamForge{users: map[string]bool{}, teams: map[string]map[string]any{}, ids: map[string]int64{}, + members: map[string][]string{}, keys: map[string][]forgeKey{}, next: 10} +} + +var ( + teamRe = regexp.MustCompile(`^/api/v1/teams/(\d+)(?:/members/([^/]+))?$`) + userKeyRe = regexp.MustCompile(`^/api/v1/(?:admin/)?users/([^/]+)/keys(?:/(\d+))?$`) +) + +func (f *teamForge) teamByID(id string) string { + for name, n := range f.ids { + if fmt.Sprint(n) == id { + return name + } + } + return "" +} + +func (f *teamForge) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + body, _ := io.ReadAll(r.Body) + var in map[string]any + _ = json.Unmarshal(body, &in) + reply := func(code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + if v != nil { + _ = json.NewEncoder(w).Encode(v) + } + } + p := r.URL.Path + if m := userKeyRe.FindStringSubmatch(p); m != nil { + switch r.Method { + case http.MethodGet: + out := []map[string]any{} + for _, k := range f.keys[m[1]] { + out = append(out, map[string]any{"id": k.ID, "title": k.Title, "key": k.Key}) + } + reply(200, out) + case http.MethodPost: + f.next++ + ro, _ := in["read_only"].(bool) + f.keys[m[1]] = append(f.keys[m[1]], forgeKey{ID: f.next, Title: in["title"].(string), Key: in["key"].(string), ReadOnly: ro}) + reply(201, map[string]any{"id": f.next}) + case http.MethodDelete: + var kept []forgeKey + for _, k := range f.keys[m[1]] { + if fmt.Sprint(k.ID) != m[2] { + kept = append(kept, k) + } + } + f.keys[m[1]] = kept + reply(204, nil) + } + return + } + if m := teamRe.FindStringSubmatch(p); m != nil { + name := f.teamByID(m[1]) + switch { + case name == "": + reply(404, map[string]any{"message": "no team"}) + case r.Method == http.MethodPatch && m[2] == "": + f.teams[name] = in + reply(200, map[string]any{}) + case r.Method == http.MethodPut: + for _, u := range f.members[name] { + if u == m[2] { + reply(204, nil) + return + } + } + f.members[name] = append(f.members[name], m[2]) + reply(204, nil) + default: + http.Error(w, "unexpected "+r.Method+" "+p, 400) + } + return + } + switch { + case r.Method == http.MethodGet && strings.HasPrefix(p, "/api/v1/users/"): + if !f.users[strings.TrimPrefix(p, "/api/v1/users/")] { + reply(404, map[string]any{"message": "no"}) + return + } + reply(200, map[string]any{"active": true}) + case r.Method == http.MethodPost && p == "/api/v1/admin/users": + f.users[in["username"].(string)] = true + reply(201, map[string]any{}) + case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodGet: + out := []map[string]any{} + for name, id := range f.ids { + out = append(out, map[string]any{"id": id, "name": name}) + } + reply(200, out) + case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodPost: + f.next++ + name := in["name"].(string) + f.ids[name], f.teams[name] = f.next, in + reply(201, map[string]any{"id": f.next}) + default: + http.Error(w, "unexpected "+r.Method+" "+p, 400) + } +} + +func (f *teamForge) titles(user string) []string { + f.mu.Lock() + defer f.mu.Unlock() + out := []string{} + for _, k := range f.keys[user] { + out = append(out, k.Title) + } + return out +} + +type teamRig struct { + t *testing.T + bus *fakeBus + forge *teamForge + svc *Service + runner *fakeRunner +} + +func newTeamRig(t *testing.T) *teamRig { + t.Helper() + forge := newTeamForge() + srv := httptest.NewServer(forge) + t.Cleanup(srv.Close) + dir := t.TempDir() + pw, host, origins := filepath.Join(dir, "pw"), filepath.Join(dir, "host"), filepath.Join(dir, "origins") + for p, c := range map[string]string{pw: "x\n", host: "novox.internal\n", origins: "https://git.novox.be\n"} { + if err := os.WriteFile(p, []byte(c), 0o600); err != nil { + t.Fatal(err) + } + } + runner := &fakeRunner{out: scanned} + svc := NewService(Config{Node: "novox", HostFile: host, OriginsFile: origins, Port: 222, + Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}, Runner: runner, Log: func(string, ...any) {}}) + svc.loaded = true + svc.loadedFrom["mesh-build-go"], svc.loadedFrom["mesh-build-node"] = true, true + return &teamRig{t: t, bus: newBus(), forge: forge, svc: svc, runner: runner} +} + +func (r *teamRig) pass() []provider.Outcome { + r.t.Helper() + out, err := r.svc.Provider(r.bus).Pass(context.Background()) + if err != nil { + r.t.Fatal(err) + } + return out +} + +// bind makes module on node bind gitea's forge-account in the assignments mirror. +func (r *teamRig) bind(node, module string) { + r.bus.set(provider.AssignmentsBucket, node+"."+module, []byte(fmt.Sprintf( + `{"node":%q,"module":%q,"provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s"}]}}`, + node, module, strings.ReplaceAll(node+"_"+module, "-", "_")))) +} + +func (r *teamRig) unbind(node, module string) { + r.bus.mu.Lock() + delete(r.bus.buckets[provider.AssignmentsBucket], node+"."+module) + r.bus.mu.Unlock() +} + +// key reports a key in module's state keys, of the kind and for the account given. +func (r *teamRig) key(module, node, account, kind, public string) { + b, _ := json.Marshal(map[string]string{"node": node, "account": account, "kind": kind, "public": public, + "fingerprint": "SHA256:x", "at": "2026-10-20T10:00:00Z"}) + r.svc.SourceKeyChanged(module, node+"."+account, b) +} + +const ( + pubGo = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG" + pubNode = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN" + pubOps = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO" +) + +// 1481/AC1: an entry from mesh-build-go on ace: the forge user nox-builder and the team builders are ensured, the team's +// units are code, read only, on every repository, and the entry's values name nox-builder. The key is registered +// read-only under its title. +func TestABuildModulesEntryIsServedAsNoxBuilderInAReadOnlyTeam(t *testing.T) { + r := newTeamRig(t) + r.bind("ace", "mesh-build-go") + r.key("mesh-build-go", "ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)") + r.pass() + + if !r.forge.users["nox-builder"] || r.forge.users["nox-agent"] { + t.Errorf("users = %v, want nox-builder alone", r.forge.users) + } + shape := r.forge.teams["builders"] + if shape == nil { + t.Fatalf("no team builders: %v", r.forge.teams) + } + units, _ := json.Marshal(shape["units_map"]) + if shape["permission"] != "read" || shape["includes_all_repositories"] != true || shape["can_create_org_repo"] != false || string(units) != `{"repo.code":"read"}` { + t.Errorf("the team's shape = %v (units %s), want code read on every repository and no other right", shape, units) + } + if got := strings.Join(r.forge.members["builders"], ","); got != "nox-builder" || len(r.forge.members["agents"]) != 0 { + t.Errorf("members = %v", r.forge.members) + } + keys := r.forge.keys["nox-builder"] + if len(keys) != 1 || keys[0].Title != "mesh:ace.mesh-build-go" || !keys[0].ReadOnly { + t.Errorf("nox-builder's keys = %+v, want one read-only key titled mesh:ace.mesh-build-go", keys) + } + e := r.bus.entry(t, "ace.mesh-build-go.forge-account") + if e.Values["user"] != "nox-builder" || e.Values["ssh_alias"] != "mesh-forge" || e.Values["host_key"] != "ssh-ed25519 AAAAC3HOSTKEY" { + t.Errorf("values = %v", e.Values) + } +} + +// 1481/AC2: an entry from ssh-client is what it was: nox-agent, the agents team, a key that may write. (The existing +// tests of the provider pass unchanged.) +func TestAnSshClientEntryIsStillTheAgentsWithAKeyThatMayWrite(t *testing.T) { + r := newTeamRig(t) + // the agents team exists, as the forge has it + r.forge.ids["agents"], r.forge.teams["agents"] = 5, map[string]any{} + r.bind("g14", "ssh-client") + r.key("ssh-client", "g14", "nox", "machine", pubA+" nox@g14 (mesh)") + r.pass() + if e := r.bus.entry(t, "g14.ssh-client.forge-account"); e.Values["user"] != "nox-agent" { + t.Errorf("values = %v", e.Values) + } + keys := r.forge.keys["nox-agent"] + if len(keys) != 1 || keys[0].Title != "mesh:g14.nox" || keys[0].ReadOnly { + t.Errorf("nox-agent's keys = %+v, want one key that may write", keys) + } + if r.forge.teams["builders"] != nil || r.forge.users["nox-builder"] { + t.Errorf("a machine's entry made the builders' account or team") + } + if got := strings.Join(r.forge.members["agents"], ","); got != "nox-agent" { + t.Errorf("agents = %v", r.forge.members) + } +} + +// 1481/AC3: a builder entry gone: only the keys titled mesh:.mesh-build-go are removed; the user and the team +// stay, and so do the other build module's key on that machine and a key of the operator's. +func TestABuilderEntryGoneRemovesItsOwnKeyAndNothingElse(t *testing.T) { + r := newTeamRig(t) + r.bind("ace", "mesh-build-go") + r.bind("ace", "mesh-build-node") + r.key("mesh-build-go", "ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)") + r.key("mesh-build-node", "ace", "mesh-build-node", "builder", pubNode+" mesh-build-node@ace (mesh)") + r.pass() + r.forge.mu.Lock() + r.forge.keys["nox-builder"] = append(r.forge.keys["nox-builder"], forgeKey{ID: 99, Title: "my own", Key: pubOps}) + r.forge.mu.Unlock() + if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-go,mesh:ace.mesh-build-node,my own" { + t.Fatalf("before: %s", got) + } + r.unbind("ace", "mesh-build-go") + r.pass() + if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-node,my own" { + t.Errorf("after: %s, want only the go module's key gone", got) + } + if !r.forge.users["nox-builder"] || r.forge.teams["builders"] == nil || len(r.forge.members["builders"]) != 1 { + t.Errorf("the user or the team went: users %v, members %v", r.forge.users, r.forge.members) + } + if r.bus.has("ace.mesh-build-go.forge-account") { + t.Errorf("the entry stayed") + } +} + +// A build module and ssh-client on one machine each have their own entry, keys, user and team. +func TestABuildModuleAndSshClientOnOneMachineDoNotShareAnything(t *testing.T) { + r := newTeamRig(t) + r.forge.ids["agents"], r.forge.teams["agents"] = 5, map[string]any{} + r.bind("ace", "ssh-client") + r.bind("ace", "mesh-build-go") + r.key("ssh-client", "ace", "nox", "machine", pubA+" nox@ace (mesh)") + r.key("mesh-build-go", "ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)") + r.pass() + if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "mesh:ace.nox" { + t.Errorf("nox-agent: %s", got) + } + if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-go" { + t.Errorf("nox-builder: %s", got) + } + // ssh-client's entry gone: nox-builder's key stays. + r.unbind("ace", "ssh-client") + r.pass() + if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-go" { + t.Errorf("nox-builder after ssh-client left: %s", got) + } + if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "" { + t.Errorf("nox-agent after ssh-client left: %s", got) + } +} + +// A build module's entry waits for its own module's builder key: a machine key of the same node, a key of the wrong +// kind in the build module's state, or the other build module's key is none. +func TestABuildModulesEntryWaitsForItsOwnBuilderKey(t *testing.T) { + r := newTeamRig(t) + r.bind("ace", "mesh-build-go") + r.key("ssh-client", "ace", "nox", "machine", pubA) // a machine's key + r.key("mesh-build-go", "ace", "mesh-build-go", "machine", pubGo) // the wrong kind in the build module's state + r.key("mesh-build-node", "ace", "mesh-build-node", "builder", pubNode) // the other build module's + out := r.pass() + if r.bus.has("ace.mesh-build-go.forge-account") || len(r.forge.users) != 0 || len(r.forge.teams) != 0 { + t.Fatalf("something was made without a builder key: users %v teams %v", r.forge.users, r.forge.teams) + } + if len(out) != 1 || !strings.Contains(out[0].Why, "mesh-build-go") || !strings.Contains(out[0].Why, "builder key") { + t.Errorf("the wait should name the module and the key: %+v", out) + } +} + +func TestTheKindOfAModulesKeyAndItsForgeUser(t *testing.T) { + for module, want := range map[string]string{"mesh-build-go": "nox-builder", "mesh-build-node": "nox-builder", "ssh-client": "nox-agent", "anything-else": "nox-agent"} { + got, err := (&Service{}).valuesForUser("ace", module) + if err != nil || got != want { + t.Errorf("%s: %q, %v; want %q", module, got, err, want) + } + } +} diff --git a/modules/gitea/cmd/forge-account-provider/main.go b/modules/gitea/cmd/forge-account-provider/main.go index 9585ac0a..8cbcec69 100644 --- a/modules/gitea/cmd/forge-account-provider/main.go +++ b/modules/gitea/cmd/forge-account-provider/main.go @@ -4,7 +4,10 @@ // For each machine whose ssh-client binds the provision and has reported its public key (ssh-client's state `keys`), // it makes the machine's agent the one forge user nox-agent (in the agents team; ADR 0410), registers the key for // it, and serves where and as whom to reach the forge over ssh: the user, the forge machine's mesh name and port, its -// host key and the two url roots. A public key is no secret, so the provision has no credential. +// host key and the two url roots. A public key is no secret, so the provision has no credential. For each build +// module that binds it and has reported its builder key (its own state `keys`), it does the same as the forge user +// nox-builder, in the builders team that reads the code of every repository and nothing else, with the key read-only +// (ADR 0466 §3). // // stdout is the MCP channel; it says what it says on stderr, and never the admin's password. package main @@ -22,6 +25,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/provider" + "gitea/internal/forgeaccount" "gitea/internal/forgehttp" ) @@ -42,11 +46,12 @@ func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte, return out, nil } -// stateKeys is ssh-client's state `keys` over the runtime (the manifest's `reads` grants it). -type stateKeys struct{} +// stateKeys is the state `keys` of one module over the runtime (the manifest's `reads` grants it): ssh-client's for +// the machines' keys, a build module's for the builders' (ADR 0466 §3). +type stateKeys struct{ module string } -func (stateKeys) Watch(handle func(key, op string, value []byte, current bool)) error { - return stdio.State("ssh-client.keys").Watch("", func(c stdio.StateChange) error { +func (k stateKeys) Watch(handle func(key, op string, value []byte, current bool)) error { + return stdio.State(k.module+".keys").Watch("", func(c stdio.StateChange) error { handle(c.Key, c.Op, c.Value, c.Current) return nil }) @@ -106,14 +111,18 @@ func main() { } } svc.OnKeys = func(string) { go pass() } - go func() { - if err := svc.WatchKeys(ctx, stateKeys{}); err != nil { - say("the machines' keys are not watched: %v", err) - return - } - say("watching the machines' keys") - pass() - }() + // Each module's keys are watched on their own, and a module whose state never answers (a build module that is not + // installed) holds back its own entries only. + for _, module := range append([]string{SSHClient}, forgeaccount.BuildModules...) { + go func() { + if err := svc.WatchSource(ctx, module, stateKeys{module}); err != nil { + say("the keys of %s are not watched: %v", module, err) + return + } + say("watching the keys of %s", module) + pass() + }() + } go func() { svc.KeepRunning(ctx, p.Run) }() diff --git a/modules/gitea/cmd/forge-account-provider/service.go b/modules/gitea/cmd/forge-account-provider/service.go index b9f33717..0eef366c 100644 --- a/modules/gitea/cmd/forge-account-provider/service.go +++ b/modules/gitea/cmd/forge-account-provider/service.go @@ -64,8 +64,11 @@ type Service struct { mu sync.Mutex keys map[string]machineKey - loaded bool // the machines' keys have been read: until then a node with none has not reported none - known string // the forge's host key line, as the machines write it into known_hosts + loaded bool // the machines' keys have been read: until then a node with none has not reported none + // loadedFrom says of each build module's state that its keys have been read, as loaded says of ssh-client's: a build + // module's state that never answers (the module is not installed) holds back its own entries and no other. + loadedFrom map[string]bool + known string // the forge's host key line, as the machines write it into known_hosts bound provider.Bus // the bus the entries are written to (set by Provider) @@ -76,14 +79,40 @@ type Service struct { type machineKey struct { Node, Account, Kind, Public string + // Source is the module whose state the key was read from: ssh-client's for a machine's, a build module's for a + // builder's. Empty is ssh-client's. + Source string } +// SSHClient is the module whose `keys` state holds the machines' keys (kind machine); the build modules' states hold +// the builders' (kind builder, ADR 0466 §2). +const SSHClient = "ssh-client" + +// sourceOf is the module a stored key was read from. +func (k machineKey) sourceOf() string { + if k.Source == "" { + return SSHClient + } + return k.Source +} + +// storageKey is the key a source's key is kept under: a machine's as it is, a builder's under its module's name. +func storageKey(source, key string) string { + if source == "" || source == SSHClient { + return key + } + return source + "/" + key +} + +// kindOf is the kind of key a consumer module's entry is made for. +func kindOf(module string) string { return forgeaccount.KindOfModule(module) } + // NewService is the service over its environment. func NewService(cfg Config) *Service { if cfg.Log == nil { cfg.Log = func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[gitea forge-account] "+f+"\n", a...) } } - return &Service{cfg: cfg, keys: map[string]machineKey{}, letGo: map[string]bool{}, Pause: time.Sleep, + return &Service{cfg: cfg, keys: map[string]machineKey{}, letGo: map[string]bool{}, loadedFrom: map[string]bool{}, Pause: time.Sleep, accounts: &forgeaccount.Accounts{Forge: cfg.Forge, Log: cfg.Log}} } @@ -103,38 +132,67 @@ func (s *Service) Provider(bus provider.Bus) *provider.Provider { } // errNoKey is a consumer whose machine has reported no key yet: nothing is made, and the client says it waits. -type errNoKey struct{ node string } +type errNoKey struct{ node, module string } func (e errNoKey) Error() string { + if kind := kindOf(e.module); kind == "builder" { + return "no key from " + e.node + " yet: its " + e.module + " has not reported a builder key, so there is no " + + "entry to make" + } return "no key from " + e.node + " yet: its ssh-client has not reported a machine key, so there is no account " + "and no entry to make" } +// sourceOfModule is the module whose state holds the keys a consumer module's entry is made from: a build module's +// own, ssh-client's for any other consumer. +func sourceOfModule(module string) string { + if kindOf(module) == "builder" { + return module + } + return SSHClient +} + +// scopeFor is the title prefix of the keys one entry registered: every key of a machine for a machine's entry, one +// build module's key for a builder's (nox-builder holds the keys of every build module of the mesh). +func scopeFor(kind, node, module string) string { + if kind == "builder" { + return forgeaccount.BuilderScope(node, module) + } + return forgeaccount.MachineScope(node) +} + // Make brings one consumer's account, its team membership and its keys into being, as the machine reported them. It // leaves what is as asked alone, so it is asked again freely. func (s *Service) Make(ctx context.Context, c provider.Consumer) error { s.making.Lock() defer s.making.Unlock() node := c.Entry.Consumer - user := forgeaccount.ForgeUser(node, "machine") + module, kind := c.Entry.Module, kindOf(c.Entry.Module) + user := forgeaccount.ForgeUser(node, kind) if user == "" { return fmt.Errorf("%q is not a machine's name an account can be named for", node) } + source := sourceOfModule(module) s.mu.Lock() loaded := s.loaded + if source != SSHClient { + loaded = s.loadedFrom[source] + } s.mu.Unlock() if !loaded { // After a restart the first pass can come before the keys arrive: nothing is removed or deleted on that. - return errors.New("the machines' keys are not read yet, so nothing is made, removed or deleted") + return fmt.Errorf("the keys of %s are not read yet, so nothing is made, removed or deleted", source) } - have := s.keysOf(node) + have := s.keysFrom(source, node, kind) if len(have) == 0 { - if err := s.releasePerNodeUser(ctx, node); err != nil { - return err + if kind == "machine" { + if err := s.releasePerNodeUser(ctx, node); err != nil { + return err + } } // A key the machine no longer reports is no longer wanted (ADR 0400 §2): the mesh's keys go, the operator's // stay, and the entry that served the machine goes with them. - if _, err := s.accounts.RemoveKeysTitled(ctx, user, forgeaccount.MachineScope(node)); err != nil { + if _, err := s.accounts.RemoveKeysTitled(ctx, user, scopeFor(kind, node, module)); err != nil { return err } if s.bound != nil { @@ -142,7 +200,7 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error { return err } } - return errNoKey{node} + return errNoKey{node: node, module: module} } if _, err := s.hostName(); err != nil { return err @@ -154,20 +212,23 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error { if err := s.accounts.EnsureAccount(ctx, user); err != nil { return err } - if err := s.accounts.EnsureInTeam(ctx, user); err != nil { + if err := s.accounts.EnsureInTeamOf(ctx, forgeaccount.TeamOfUser(user), user); err != nil { return err } var wanted []forgeaccount.WantedKey for _, k := range have { - wanted = append(wanted, forgeaccount.WantedKey{Title: forgeaccount.KeyTitle(k.Node, k.Account), Key: k.Public}) + // A builder's key may read and never push (ADR 0466 §3): the key says it beside the team that does. + wanted = append(wanted, forgeaccount.WantedKey{Title: forgeaccount.KeyTitle(k.Node, k.Account), Key: k.Public, ReadOnly: kind == "builder"}) } // The forge holds one public key on one user only, so the key leaves agent- before it joins nox-agent. That // is done here, after everything that can fail before it, so the window in which the machine's key is on no user // is only the span between these two calls; if the add fails, the next pass (at most ten minutes) puts it back. - if err := s.releasePerNodeUser(ctx, node); err != nil { - return err + if kind == "machine" { + if err := s.releasePerNodeUser(ctx, node); err != nil { + return err + } } - if err := s.accounts.EnsureKeys(ctx, user, forgeaccount.MachineScope(node), wanted); err != nil { + if err := s.accounts.EnsureKeys(ctx, user, scopeFor(kind, node, module), wanted); err != nil { return err } s.mu.Lock() @@ -176,20 +237,25 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error { return nil } -// Gone removes the keys the mesh registered for a machine that no longer binds the forge. The account stays: it may +// Gone removes the keys the mesh registered for a machine (or, for a build module, the module's key on that machine) +// that no longer binds the forge. The account stays: it may // hold what the machine's agent wrote, and the keys of the operator's own that were added by hand are not touched // (ADR 0399 §2). func (s *Service) Gone(ctx context.Context, e provider.Entry) error { s.making.Lock() defer s.making.Unlock() - user := forgeaccount.ForgeUser(e.Consumer, "machine") + kind := kindOf(e.Module) + user := forgeaccount.ForgeUser(e.Consumer, kind) if user == "" { return nil } - if err := s.releasePerNodeUser(ctx, e.Consumer); err != nil { - return err + if kind == "machine" { + if err := s.releasePerNodeUser(ctx, e.Consumer); err != nil { + return err + } } - _, err := s.accounts.RemoveKeysTitled(ctx, user, forgeaccount.MachineScope(e.Consumer)) + // A builder's entry gone removes that entry's key only: the user and the team serve the other machines. + _, err := s.accounts.RemoveKeysTitled(ctx, user, scopeFor(kind, e.Consumer, e.Module)) return err } @@ -220,17 +286,24 @@ func (s *Service) releasePerNodeUser(ctx context.Context, node string) error { } // KeyChanged takes in one machine key as the bus keeps it, and tells OnKeys its node. -func (s *Service) KeyChanged(key string, value []byte) { - if node := s.store(key, value); node != "" && s.OnKeys != nil { +func (s *Service) KeyChanged(key string, value []byte) { s.SourceKeyChanged(SSHClient, key, value) } + +// KeyGone forgets one machine key, and tells OnKeys its node. +func (s *Service) KeyGone(key string) { s.SourceKeyGone(SSHClient, key) } + +// SourceKeyChanged takes in one key of the state `keys` of the module source (ssh-client's, or a build module's), and +// tells OnKeys its node. +func (s *Service) SourceKeyChanged(source, key string, value []byte) { + if node := s.storeFrom(source, key, value); node != "" && s.OnKeys != nil { s.OnKeys(node) } } -// KeyGone forgets one machine key, and tells OnKeys its node. -func (s *Service) KeyGone(key string) { +// SourceKeyGone forgets one key of the module source's state, and tells OnKeys its node. +func (s *Service) SourceKeyGone(source, key string) { node, _, _ := strings.Cut(key, ".") s.mu.Lock() - delete(s.keys, key) + delete(s.keys, storageKey(source, key)) s.mu.Unlock() if s.OnKeys != nil { s.OnKeys(node) @@ -240,24 +313,38 @@ func (s *Service) KeyGone(key string) { // store keeps a machine key that reads as one: its node and account the key's own, and a public half that is a // public key line and nothing private. It answers the node, or "" for a value it did not keep (and drops what was // kept under that key, since the key now says something else). -func (s *Service) store(key string, value []byte) string { +func (s *Service) store(key string, value []byte) string { return s.storeFrom(SSHClient, key, value) } + +// storeFrom is store for a key of the module source's state. +func (s *Service) storeFrom(source, key string, value []byte) string { node, account, _ := strings.Cut(key, ".") + stored := storageKey(source, key) var k machineKey var raw struct { Node, Account, Kind, Public string } err := json.Unmarshal(value, &raw) - k = machineKey(raw) + k = machineKey{Node: raw.Node, Account: raw.Account, Kind: raw.Kind, Public: raw.Public} + if source != SSHClient { + k.Source = source + } s.mu.Lock() defer s.mu.Unlock() - if err != nil || k.Node != node || k.Account != account || !publicKeyLine(k.Public) { - if _, had := s.keys[key]; had { - delete(s.keys, key) - } - s.cfg.Log("the key %s is not a machine's public key; it is not registered", key) + if want := kindOf(source); err == nil && k.Kind != want && k.Node == node && k.Account == account { + // Each state holds one kind: ssh-client's the machines', a build module's the builders' (ADR 0466 §2). The other + // kind in either is not this entry's to serve, and is said. + delete(s.keys, stored) + s.cfg.Log("the key %s in the state keys of %s is of kind %q, not %q; it is ignored", key, source, k.Kind, want) return "" } - s.keys[key] = k + if err != nil || k.Node != node || k.Account != account || !publicKeyLine(k.Public) { + if _, had := s.keys[stored]; had { + delete(s.keys, stored) + } + s.cfg.Log("the key %s of %s is not a public key that can be registered; it is not registered", key, source) + return "" + } + s.keys[stored] = k return node } @@ -267,12 +354,15 @@ func publicKeyLine(line string) bool { } // keysOf are the machine keys one node reported, by key. -func (s *Service) keysOf(node string) []machineKey { +func (s *Service) keysOf(node string) []machineKey { return s.keysFrom(SSHClient, node, "machine") } + +// keysFrom are the keys of the kind that one node reported in the module source's state, by key. +func (s *Service) keysFrom(source, node, kind string) []machineKey { s.mu.Lock() defer s.mu.Unlock() var names []string for key, k := range s.keys { - if k.Node == node && k.Kind == "machine" && strings.HasPrefix(key, node+".") { + if k.Node == node && k.Kind == kind && k.sourceOf() == source && strings.HasPrefix(key, storageKey(source, node+".")) { names = append(names, key) } } @@ -284,27 +374,37 @@ func (s *Service) keysOf(node string) []machineKey { return out } -// WatchKeys keeps the machines' keys as the bus holds them, asking again until the runtime answers (the channel to it -// opens as the bundle starts, and the state of ssh-client may arrive after this one). A change after the first -// values is told to OnKeys; the first values are not, since one pass answers them all. +// WatchKeys keeps the machines' keys as the bus holds them (ssh-client's state `keys`), asking again until the runtime +// answers (the channel to it opens as the bundle starts, and the state of ssh-client may arrive after this one). A +// change after the first values is told to OnKeys; the first values are not, since one pass answers them all. func (s *Service) WatchKeys(ctx context.Context, src KeySource) error { + return s.WatchSource(ctx, SSHClient, src) +} + +// WatchSource is WatchKeys for the state `keys` of any module whose keys the provider registers: ssh-client's, or a +// build module's (ADR 0466 §3). Once its first values are in, the entries made from them are no longer held back. +func (s *Service) WatchSource(ctx context.Context, source string, src KeySource) error { waits := []time.Duration{2 * time.Second, 5 * time.Second, 10 * time.Second, 30 * time.Second} for attempt := 0; ; attempt++ { err := src.Watch(func(key, op string, value []byte, current bool) { switch { case op != "put" && current: case op != "put": - s.KeyGone(key) + s.SourceKeyGone(source, key) case current: - s.store(key, value) + s.storeFrom(source, key, value) default: - s.KeyChanged(key, value) + s.SourceKeyChanged(source, key, value) } }) if err == nil { // The watch answers once the current values are all handled (stdio KeptState.Watch). s.mu.Lock() - s.loaded = true + if source == SSHClient { + s.loaded = true + } else { + s.loadedFrom[source] = true + } s.mu.Unlock() return nil } @@ -312,7 +412,7 @@ func (s *Service) WatchKeys(ctx context.Context, src KeySource) error { if attempt < len(waits) { pause = waits[attempt] } - s.cfg.Log("the machines' keys cannot be watched yet (%v); asking again in %s", err, pause) + s.cfg.Log("the keys of %s cannot be watched yet (%v); asking again in %s", source, err, pause) if ctx.Err() != nil { return ctx.Err() } @@ -377,9 +477,18 @@ func knownHostLine(scanned, host string, port int) (string, error) { return "", errors.New("the forge's sshd gave no ed25519 host key") } +// valuesForUser is the forge user a consumer module's entry is served as. +func (s *Service) valuesForUser(node, module string) (string, error) { + user := forgeaccount.ForgeUser(node, kindOf(module)) + if user == "" { + return "", fmt.Errorf("%q is not a machine's name an account can be named for", node) + } + return user, nil +} + // valuesFor are the values served to one machine's consumer (ADR 0400 §3, §4). -func (s *Service) valuesFor(node string) (map[string]any, error) { - user := forgeaccount.ForgeUser(node, "machine") +func (s *Service) valuesFor(node, module string) (map[string]any, error) { + user := forgeaccount.ForgeUser(node, kindOf(module)) host, err := s.hostName() if err != nil { return nil, err @@ -421,7 +530,7 @@ func (b *valuesBus) Put(ctx context.Context, bucket, key string, value []byte) e return err } if e.Refused == "" { - extra, err := b.s.valuesFor(e.Consumer) + extra, err := b.s.valuesFor(e.Consumer, e.Module) if err != nil { return err } diff --git a/modules/gitea/cmd/forge-account-provider/watch_test.go b/modules/gitea/cmd/forge-account-provider/watch_test.go new file mode 100644 index 00000000..e8986788 --- /dev/null +++ b/modules/gitea/cmd/forge-account-provider/watch_test.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "slices" + "strings" + "sync" + "testing" + "time" +) + +// Issue 1482 (novox/hq ADR 0466 §3): the provider watches the build modules' state `keys` beside ssh-client's, and +// registers each builder entry read-only on nox-builder. + +func keyJSON(node, account, kind, public string) []byte { + b, _ := json.Marshal(map[string]string{"node": node, "account": account, "kind": kind, "public": public, + "fingerprint": "SHA256:x", "at": "2026-10-20T10:00:00Z"}) + return b +} + +// watched is a team rig whose keys come through watches of the modules' states, as in the running provider. +type watched struct { + *teamRig + sources map[string]*fakeKeys + passes *int + logs *[]string + mu *sync.Mutex +} + +func newWatched(t *testing.T) *watched { + r := newTeamRig(t) + r.svc.loaded, r.svc.loadedFrom = false, map[string]bool{} + r.svc.Pause = func(time.Duration) {} + n, logs, mu := 0, []string{}, &sync.Mutex{} + r.svc.OnKeys = func(string) { n++ } + r.svc.cfg.Log = func(f string, a ...any) { mu.Lock(); logs = append(logs, fmt.Sprintf(f, a...)); mu.Unlock() } + w := &watched{teamRig: r, sources: map[string]*fakeKeys{}, passes: &n, logs: &logs, mu: mu} + for _, module := range []string{"ssh-client", "mesh-build-go", "mesh-build-node"} { + w.sources[module] = &fakeKeys{} + if err := r.svc.WatchSource(context.Background(), module, w.sources[module]); err != nil { + t.Fatal(err) + } + } + return w +} + +func (w *watched) said() string { + w.mu.Lock() + defer w.mu.Unlock() + return strings.Join(*w.logs, "\n") +} + +// 1482/AC1: a mesh-build-go.keys entry ace.mesh-build-go of kind builder: its public half is registered read-only on +// nox-builder titled mesh:ace.mesh-build-go. +func TestABuilderEntryOfABuildModulesStateIsRegisteredReadOnlyOnNoxBuilder(t *testing.T) { + w := newWatched(t) + w.bind("ace", "mesh-build-go") + w.sources["mesh-build-go"].send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)"), true) + w.pass() + keys := w.forge.keys["nox-builder"] + if len(keys) != 1 || keys[0].Title != "mesh:ace.mesh-build-go" || !keys[0].ReadOnly || keys[0].Key != pubGo+" mesh-build-go@ace (mesh)" { + t.Errorf("nox-builder's keys = %+v", keys) + } + if !w.svc.loadedFrom["mesh-build-go"] { + t.Errorf("the state was watched and its keys are not counted as read") + } +} + +// 1482/AC2: that entry changed: the old key is removed and the new one registered; deleted: the key is removed and no key +// titled otherwise is touched. +func TestAChangedBuilderEntryReplacesItsKeyAndADeletedOneRemovesIt(t *testing.T) { + w := newWatched(t) + w.bind("ace", "mesh-build-go") + src := w.sources["mesh-build-go"] + src.send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "builder", pubGo), true) + w.pass() + w.forge.mu.Lock() + w.forge.keys["nox-builder"] = append(w.forge.keys["nox-builder"], forgeKey{ID: 99, Title: "my own", Key: pubOps}) + w.forge.mu.Unlock() + + before := *w.passes + src.send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "builder", pubNode), false) + if *w.passes != before+1 { + t.Errorf("a change asked for %d passes, want 1", *w.passes-before) + } + w.pass() + got := w.forge.keys["nox-builder"] + if len(got) != 2 || got[0].Title != "my own" || got[1].Title != "mesh:ace.mesh-build-go" || got[1].Key != pubNode || !got[1].ReadOnly { + t.Errorf("after the change: %+v", got) + } + + src.send("ace.mesh-build-go", "delete", nil, false) + w.pass() + if got := strings.Join(w.forge.titles("nox-builder"), ","); got != "my own" { + t.Errorf("after the delete: %s, want only the operator's key", got) + } +} + +// 1482/AC3: an entry of kind machine in a build module's state, or of kind builder in ssh-client's, is ignored and +// said in the provider's log. +func TestAKeyOfTheWrongKindForItsStateIsIgnoredAndSaid(t *testing.T) { + w := newWatched(t) + w.bind("ace", "mesh-build-go") + w.bind("g14", "ssh-client") + w.sources["mesh-build-go"].send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "machine", pubGo), true) + w.sources["ssh-client"].send("g14.nox", "put", keyJSON("g14", "nox", "builder", pubA), true) + w.pass() + if len(w.forge.users) != 0 || w.bus.has("ace.mesh-build-go.forge-account") || w.bus.has("g14.ssh-client.forge-account") { + t.Errorf("a key of the wrong kind was served: users %v", w.forge.users) + } + said := w.said() + for _, want := range []string{`ace.mesh-build-go in the state keys of mesh-build-go is of kind "machine", not "builder"`, `g14.nox in the state keys of ssh-client is of kind "builder", not "machine"`} { + if !strings.Contains(said, want) { + t.Errorf("the log lacks %q:\n%s", want, said) + } + } +} + +// A build module's state that never answers holds back its own entries, never a machine's. +func TestABuildModulesStateThatNeverAnswersHoldsBackOnlyItsOwnEntries(t *testing.T) { + w := newWatched(t) + w.svc.loadedFrom = map[string]bool{} // mesh-build-go's watch has not answered + w.bind("ace", "mesh-build-go") + w.bind("g14", "ssh-client") + w.sources["ssh-client"].send("g14.nox", "put", keyJSON("g14", "nox", "machine", pubA+" nox@g14 (mesh)"), true) + out := w.pass() + if !w.bus.has("g14.ssh-client.forge-account") { + t.Errorf("a machine's entry waited for a build module's state") + } + waits := 0 + for _, o := range out { + if strings.Contains(o.Key, "mesh-build-go") && strings.Contains(o.Why, "keys of mesh-build-go are not read yet") { + waits++ + } + } + if waits != 1 { + t.Errorf("outcomes = %+v", out) + } +} + +// 1482/AC4 (the provider's half): the manifest reads the states of both build modules beside ssh-client's and the +// controller's assignments; the provider watches exactly the modules the code names as builders. +func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Reads []string `json:"reads"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys"} { + if !slices.Contains(m.Reads, state) { + t.Errorf("gitea's reads %v lack %q", m.Reads, state) + } + } +} diff --git a/modules/gitea/internal/forgeaccount/account.go b/modules/gitea/internal/forgeaccount/account.go index 0b3af3d0..8fe9a721 100644 --- a/modules/gitea/internal/forgeaccount/account.go +++ b/modules/gitea/internal/forgeaccount/account.go @@ -1,7 +1,8 @@ // Package forgeaccount is the forge accounts the mesh makes for its agents (novox/hq ADR 0365 clauses 1 and 3, ADR // 0398 §1, ADR 0399, ADR 0400; issue 955): an account of the agent's own, no administrator, in the agents write // team. Both the token tool (cmd/agent-token) and the forge-account provider use it, so there is one place that says -// what an agent's account is. +// what an agent's account is. It also makes the one account of the build modules' keys, nox-builder, in the builders +// team that reads the code of every repository and nothing else (ADR 0466 §3; issue 1481). package forgeaccount import ( @@ -26,8 +27,23 @@ const ( TeamName = "agents" ) -// teamShape is what the agents team grants: write on code, issues and pull requests of every repository of the +// BuildersTeamName is the team of the build modules' account (ADR 0466 §3). +const BuildersTeamName = "builders" + +// Team is a team of the organisation as the mesh keeps it: its name and the shape it is put back to. +type Team struct { + Name string + Shape func() map[string]any +} + +// AgentsTeam is the team of the machines' agents: write on code, issues and pull requests of every repository of the // organisation and no more, the shape of the forge tools' team (tools-account.ts). +var AgentsTeam = Team{Name: TeamName, Shape: teamShape} + +// BuildersTeam is the team of the build modules' account: it may read the code of every repository of the organisation +// and has no other right (ADR 0466 §3). +var BuildersTeam = Team{Name: BuildersTeamName, Shape: buildersTeamShape} + func teamShape() map[string]any { return map[string]any{ "name": TeamName, "permission": "write", "includes_all_repositories": true, "can_create_org_repo": false, @@ -35,6 +51,13 @@ func teamShape() map[string]any { } } +func buildersTeamShape() map[string]any { + return map[string]any{ + "name": BuildersTeamName, "permission": "read", "includes_all_repositories": true, "can_create_org_repo": false, + "units_map": map[string]string{"repo.code": "read"}, + } +} + // User is gitea's account as the admin reads it. type User struct { IsAdmin bool `json:"is_admin"` @@ -87,35 +110,41 @@ func (m *Accounts) EnsureAccount(ctx context.Context, account string) error { // EnsureInTeam makes the agents team when there is none, puts a team that is there back to its shape, and adds the // account to it; an account already in it stays (ADR 0398 §1, issue 948). func (m *Accounts) EnsureInTeam(ctx context.Context, account string) error { - id, err := m.teamID(ctx) + return m.EnsureInTeamOf(ctx, AgentsTeam, account) +} + +// EnsureInTeamOf is EnsureInTeam for any team of the mesh's: it makes the team when there is none, puts a team that is +// there back to its shape, and adds the account to it. +func (m *Accounts) EnsureInTeamOf(ctx context.Context, team Team, account string) error { + id, err := m.teamID(ctx, team.Name) if err != nil { return err } if id != 0 { - if err := m.Forge.SendJSON(ctx, http.MethodPatch, fmt.Sprintf("/api/v1/teams/%d", id), teamShape(), nil); err != nil { - return fmt.Errorf("the team %s could not be put back to its shape: %w", TeamName, err) + if err := m.Forge.SendJSON(ctx, http.MethodPatch, fmt.Sprintf("/api/v1/teams/%d", id), team.Shape(), nil); err != nil { + return fmt.Errorf("the team %s could not be put back to its shape: %w", team.Name, err) } } else { var made struct { ID int64 `json:"id"` } - if err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/orgs/"+TeamOrg+"/teams", teamShape(), &made); err != nil { - return fmt.Errorf("the team %s could not be made: %w", TeamName, err) + if err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/orgs/"+TeamOrg+"/teams", team.Shape(), &made); err != nil { + return fmt.Errorf("the team %s could not be made: %w", team.Name, err) } id = made.ID - m.say("made the forge team %s in %s: write on every repository", TeamName, TeamOrg) + m.say("made the forge team %s in %s", team.Name, TeamOrg) } if id == 0 { - return fmt.Errorf("the forge answered no id for the team %s", TeamName) + return fmt.Errorf("the forge answered no id for the team %s", team.Name) } if err := m.Forge.SendJSON(ctx, http.MethodPut, fmt.Sprintf("/api/v1/teams/%d/members/%s", id, account), nil, nil); err != nil { - return fmt.Errorf("the account %s could not be added to the team %s: %w", account, TeamName, err) + return fmt.Errorf("the account %s could not be added to the team %s: %w", account, team.Name, err) } return nil } -// teamID is the agents team's id, or 0 when there is none. -func (m *Accounts) teamID(ctx context.Context) (int64, error) { +// teamID is the id of the team of that name, or 0 when there is none. +func (m *Accounts) teamID(ctx context.Context, name string) (int64, error) { var id int64 for page := 1; page <= 20 && id == 0; page++ { var teams []struct { @@ -126,7 +155,7 @@ func (m *Accounts) teamID(ctx context.Context) (int64, error) { return 0, fmt.Errorf("the teams of %s could not be read: %w", TeamOrg, err) } for _, t := range teams { - if t.Name == TeamName { + if t.Name == name { id = t.ID } } @@ -160,16 +189,49 @@ var NodeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`) // AgentUser is the one forge user of every machine's agents (ADR 0410 §1), in the agents write team. const AgentUser = "nox-agent" -// ForgeUser is the forge user of a machine's key of the given kind (ADR 0410 §1): `nox-agent` for every machine -// and account (kind machine). A kind not built yet has none, and the answer is empty. The machine's name must still -// be one an account could be named for. +// BuilderUser is the one forge user of every build module's key (ADR 0466 §3), in the builders team: it reads and +// pushes nothing. +const BuilderUser = "nox-builder" + +// BuildModules are the modules whose keys are the builders' (kind builder), by their name, which is also the name of +// the account each runs as (ADR 0466 §1). +var BuildModules = []string{"mesh-build-go", "mesh-build-node"} + +// KindOfModule is the kind of key a consumer module's entry is made for: builder for a build module, machine for any +// other (ssh-client's). +func KindOfModule(module string) string { + for _, m := range BuildModules { + if m == module { + return "builder" + } + } + return "machine" +} + +// ForgeUser is the forge user of a key of the given kind (ADR 0410 §1, ADR 0466 §3): `nox-agent` for every machine +// and account (kind machine), `nox-builder` for every build module (kind builder). A kind not built yet has none, and +// the answer is empty. The machine's name must still be one an account could be named for. func ForgeUser(node, kind string) string { - if kind == "machine" && NodeName.MatchString(node) { + if !NodeName.MatchString(node) { + return "" + } + switch kind { + case "machine": return AgentUser + case "builder": + return BuilderUser } return "" } +// TeamOfUser is the team a forge user of the mesh is in. +func TeamOfUser(user string) Team { + if user == BuilderUser { + return BuildersTeam + } + return AgentsTeam +} + // NodeAccount is the forge account the token tool (cmd/agent-token) keeps for a node's agent: `agent-` (ADR // 0365 §1). It is not the forge user of the machine's ssh key (ForgeUser), and its behaviour is unchanged. func NodeAccount(node string) string { return AccountPrefix + node } diff --git a/modules/gitea/internal/forgeaccount/builder_test.go b/modules/gitea/internal/forgeaccount/builder_test.go new file mode 100644 index 00000000..fb920dfd --- /dev/null +++ b/modules/gitea/internal/forgeaccount/builder_test.go @@ -0,0 +1,46 @@ +package forgeaccount + +import "testing" + +// Issue 1481 (novox/hq ADR 0466 §3): the build modules' forge user, team and key scope. +func TestTheBuildModulesHaveOneForgeUserAndTheMachinesKeepTheirs(t *testing.T) { + for kind, want := range map[string]string{"machine": "nox-agent", "builder": "nox-builder", "session": ""} { + if got := ForgeUser("ace", kind); got != want { + t.Errorf("ForgeUser(ace, %s) = %q, want %q", kind, got, want) + } + } + if ForgeUser("Not A Node", "builder") != "" { + t.Errorf("a name no account could be named for got a user") + } + for module, want := range map[string]string{"mesh-build-go": "builder", "mesh-build-node": "builder", "ssh-client": "machine", "": "machine"} { + if got := KindOfModule(module); got != want { + t.Errorf("KindOfModule(%q) = %s, want %s", module, got, want) + } + } + if TeamOfUser(BuilderUser).Name != "builders" || TeamOfUser(AgentUser).Name != "agents" { + t.Errorf("teams: %s and %s", TeamOfUser(BuilderUser).Name, TeamOfUser(AgentUser).Name) + } +} + +// A build module's scope names the module, so one module's pass does not remove the other's key on the same machine; +// the machine's scope is every key of the machine. +func TestABuilderScopeNamesTheModule(t *testing.T) { + if got := BuilderScope("ace", "mesh-build-go"); got != "mesh:ace.mesh-build-go" { + t.Errorf("scope = %q", got) + } + if BuilderScope("ace", "mesh-build-go") == BuilderScope("ace", "mesh-build-node") { + t.Errorf("two build modules share a scope") + } + if got := KeyTitle("ace", "mesh-build-go"); got != BuilderScope("ace", "mesh-build-go") { + t.Errorf("the key's title %q is not inside the scope", got) + } +} + +// The builders team reads the code of every repository of the organisation and has no other right. +func TestTheBuildersTeamReadsCodeOnly(t *testing.T) { + s := BuildersTeam.Shape() + units, _ := s["units_map"].(map[string]string) + if s["permission"] != "read" || s["includes_all_repositories"] != true || s["can_create_org_repo"] != false || len(units) != 1 || units["repo.code"] != "read" { + t.Errorf("shape = %v", s) + } +} diff --git a/modules/gitea/internal/forgeaccount/keys.go b/modules/gitea/internal/forgeaccount/keys.go index 68387ea5..84cd9e08 100644 --- a/modules/gitea/internal/forgeaccount/keys.go +++ b/modules/gitea/internal/forgeaccount/keys.go @@ -22,6 +22,8 @@ type WantedKey struct { Title string // Key is the public key line: type, base64 and a comment. Key string + // ReadOnly registers the key as one that may not push (a builder's, ADR 0466 §3). + ReadOnly bool } // Key is a key the forge holds for an account. @@ -56,10 +58,10 @@ func (m *Accounts) Keys(ctx context.Context, user string) ([]Key, error) { return all, nil } -// AddKey registers a key for the account, one it may write with. +// AddKey registers a key for the account, one it may write with unless the key is wanted read-only. func (m *Accounts) AddKey(ctx context.Context, user string, k WantedKey) error { err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/admin/users/"+user+"/keys", - map[string]any{"title": k.Title, "key": k.Key, "read_only": false}, nil) + map[string]any{"title": k.Title, "key": k.Key, "read_only": k.ReadOnly}, nil) if err != nil { return fmt.Errorf("the key %s could not be added for %s: %w", k.Title, user, err) } @@ -78,6 +80,11 @@ func (m *Accounts) DeleteKey(ctx context.Context, user string, id int64) error { // MachineScope is the title prefix of one machine's keys, `mesh:.`: what a pass for that machine may remove. func MachineScope(node string) string { return TitlePrefix + node + "." } +// BuilderScope is the title prefix of one build module's key on one machine, `mesh:.`: what a pass for +// that module's entry may remove. It names the module, since nox-builder holds the keys of both build modules of a +// machine and one's pass must not remove the other's. +func BuilderScope(node, module string) string { return TitlePrefix + node + "." + module } + // EnsureKeys registers each wanted key the account does not hold and removes the keys the mesh registered // for one machine (title starting scope, `mesh:.`) that are no longer wanted. One forge user holds every // machine's keys (ADR 0410 §2), so a key under another machine's title is never removed here. A key whose title does not start `mesh:` is the operator's and diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 2de97308..118224cd 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -540,6 +540,8 @@ ], "reads": [ "mesh-controller.assignments", - "ssh-client.keys" + "ssh-client.keys", + "mesh-build-go.keys", + "mesh-build-node.keys" ] } diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md new file mode 100644 index 00000000..61433b7e --- /dev/null +++ b/modules/mesh-build-go/README.md @@ -0,0 +1,88 @@ +# mesh-build-go — the Go build module + +Design 47, release R4c-3 (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470). The module that builds and checks Go +code on a build machine, as an account of its own (`runs-as`, no login shell, home at mode 0700), with the +toolchain installed as the packages it declares (`go`, `git`, `openssh`, `postgresql`) and the capability +`user-namespaces`. It claims no seat: it is assigned to the build machines (ADR 0469 §6) and runs dark until the +builder uses it. + +## The recipe + +`cmd/mesh-build-go/recipe.go` names the one Go version the module builds with (`RecipeGo`), the postgres major +version the integration level tests with, and the recipe's own version. The recipe version is a hash of those and of +the nats-server library's version, read from the program's build information, so a change of any of them moves it. + +## The guard and the health + +At start, every ten minutes, before every step and at every health look, the module reads the installed Go +(`go version`) and writes it with the recipe version to `.go` of its `toolchains` state. A machine whose Go +is not the recipe's refuses every step, the step ending `refused` on its key in `steps` with both versions named, +and `build_go_health` (the module's `runtime-health`) answers unhealthy with the same text, so the controller raises +the machine's condition. Moving to a new Go is a change of `RecipeGo`. + +## Tools + +`build_go_status`, `build_go_health` and `build_go_fetch` are served now; the other step tools (`build_go_test`, +`build_go_digest`, `build_go_build`) are added by their own issues to `StepTools` and run behind the guard: a long +step answers at once with its id and ends on `.` of `steps` (ADR 0469 §2); the digest answers in its call. + +## Fetch + +`build_go_fetch {repository, commit, ref?}` fetches the commit over SSH as the module's account (`mesh-forge` is the +forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the +work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor, +with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an +environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day +are removed. + +## Test + +`build_go_test {workspace, level, packages?}` runs the tests of a fetched workspace by Go's convention, one level at a +time. The unit level is `go test -json` over every package without the `integration` build tag, in each directory +with a `go.mod`, judged by its one bound: its running time, that of the slowest package (compile not counted), within +five seconds. The step ends passed when the level ran, and its result says whether the level did: `passed`, the +packages with their times, the tests that failed with their first lines of output, the five slowest tests, and `why` +when it failed (a failed test, a build that failed) or ran over a bound. `passed` is the tests' own verdict: a level over its bound still passes and says `over_bound` (and `over_which`, setup or run) with the slowest tests, and the checker judges the bounds (ADR 0453 §9). A step ends failed only when go test could not +run. A repository with no tests at the level has none, and the result says so: that is a pass. `testdata/captured` holds +the output of the real go for the fixture repositories in `testdata/repos`; the unit tests read it, and the +integration-tagged test runs the real go on the fixtures. + +## The integration level + +`build_go_test` with `level: integration` runs `go test -tags integration` against a store and a bus started for the +run alone (novox/hq ADR 0470). The store is a postgres of the installed package: `initdb` and `pg_ctl` into +`//run-/pg`, as the module's account, listening on a unix socket there and on no TCP address, with +durability off; the tests get `MESH_TEST_POSTGRES` (the socket directory percent-encoded in the URL, because the +controller's tests cut the URL at its last slash). The bus is a nats-server from the Go library at the version this +module's `go.mod` names, run by this same binary started as `mesh-build-go bus ` (`bus.go`: JetStream on, a +loopback port of its own, its address on the first line, stopped when its input closes); the tests get `MESH_TEST_NATS`. +Both run inside the step, are stopped, and the run's directory is removed whatever the tests did. + +Setup (compiling the tests, the store, the bus) is judged within 30 s and the tests' running time (the wall time of +`go test`) within 60 s, apart; a level over either says `over_bound` and names the bound, with the slowest tests for the running +time. A machine whose installed postgres major is not the recipe's refuses this level, naming both; the unit level +still runs. The result names the postgres major and the nats-server version it ran against. The mesh's own store major +is not named: no state or grant gives it to this module (ADR 0470 §4 asks for it when it differs), so only the recipe's +is said until a source exists. + +## Digest + +`build_go_digest {workspace, level}` answers in its call, for each package of the workspace that has tests at the level, +one digest of everything those tests read (ADR 0445 §5): the files, by content and under their path in the tree, of the +package and every package its test binary reaches (`go list -test -deps`) that lies in the module's own tree; the version +and checksum of every module outside it; the package's own `testdata/` files; and the Go version, the level, the build tags +and, at the integration level, the postgres major and the nats-server version. Packages of the standard library are the Go +version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors +is listed and tested from its vendor directory (`-mod=vendor`). + +## Build + +`build_go_build {workspace, module, artifact}` builds one Go bundle artifact of a module of the workspace with the build +agent's recipe (mesh-controller's `toolchain.go`, the Go entry, copied in `testdata/captured/toolchain-go-entry.txt`, and +`builder.go`): `CGO_ENABLED=0`, `GOFLAGS=-trimpath -buildvcs=false` (with `-mod=mod`, or `-mod=vendor` for a directory +that vendors), `go build -ldflags "-s -w -X main.builtFor=" -o .mesh-build// ./` in the +module's directory, by the installed Go (`GOTOOLCHAIN=local`) from the module cache the fetch filled and no network +(`GOPROXY=off`). The binary is named as the build agent names it. The output directory is packed with mesh-sdk's `Pack`, +so the digest is the build agent's for the same tree. The step ends with the artifact, the binary, the digest, the size and +the path of the archive, which stays in the workspace's `artifacts` directory: publishing it is the builder's, in a later +release. An artifact that is not a Go bundle fails the step naming it. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build.go b/modules/mesh-build-go/cmd/mesh-build-go/build.go new file mode 100644 index 00000000..b18915bf --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/build.go @@ -0,0 +1,164 @@ +package main + +// The build step (novox/hq ADR 0405 §3 §7, ADR 0453 §7; issue 1487): a Go bundle artifact of a module of a fetched +// workspace, built with today's recipe — the build agent's, mesh-controller's internal/builder/toolchain.go (the Go +// entry) and builder.go (compile, binaryName, pack) — by the installed Go and no network, packed with mesh-sdk's Pack +// so that its digest is the build agent's for the same tree. The archive is left in the workspace; publishing it is +// the builder's, in a later release. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The Go recipe of the build agent (toolchain.go, the entry of language go): static, trimmed of its own paths and of +// the commit stamp, stripped of symbols and debug info, and told the system it was built for. +const ( + recipeFlags = "-trimpath -buildvcs=false" + recipeLinkerFlags = "-s -w" + recipeSystemStamp = "main.builtFor" + // outDir is where a build leaves an artifact in the module's directory (builder.Out). + outDir = ".mesh-build" +) + +// BuildResult is what a passed build step says. +type BuildResult struct { + Artifact string `json:"artifact"` + Binary string `json:"binary"` + Digest string `json:"digest"` + Size int `json:"size"` + // Path is the archive, in the workspace's directory of the work directory. + Path string `json:"path"` +} + +func (m *Module) buildTool() StepTool { + return StepTool{ + Name: "build_go_build", + Description: "Build one Go bundle artifact of a module of a fetched workspace with the build agent's recipe, by the " + + "installed Go and no network, and pack it. Answers a step id at once; the step ends on its key with the " + + "artifact's digest, size and the archive's path.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, + "module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"}, + "artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"}, + }, + "required": []string{"workspace", "module", "artifact"}, + }, + Long: true, + Run: m.build, + } +} + +// binaryName is what a compiled bundle's executable is called (builder.go binaryName): what the artifact says, or the +// name of the package it is built from, or the artifact's. +func binaryName(a catalogue.Artifact) string { + if name := catalogue.BinaryOf(a); name != "" { + return name + } + if name := strings.TrimSpace(a.Binary); name != "" { + return name + } + if from := strings.Trim(a.From, "./"); from != "" { + return filepath.Base(from) + } + return a.Name +} + +func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { + workspace, _ := args["workspace"].(string) + module, _ := args["module"].(string) + name, _ := args["artifact"].(string) + if !workspaceOK.MatchString(workspace) { + return nil, fmt.Errorf("%q is not a workspace id", workspace) + } + tree := filepath.Join(m.Work, workspace, "tree") + dir := filepath.Join(tree, filepath.FromSlash(module)) + if rel, err := filepath.Rel(tree, dir); err != nil || strings.HasPrefix(rel, "..") { + return nil, fmt.Errorf("the module directory %q lies outside the workspace", module) + } + raw, err := os.ReadFile(filepath.Join(dir, "module.json")) + if err != nil { + return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err) + } + manifest, err := catalogue.ParseManifest(raw) + if err != nil { + return nil, fmt.Errorf("the manifest in %q: %w", module, err) + } + var art *catalogue.Artifact + names := []string{} + if manifest.Build != nil { + for i := range manifest.Build.Artifacts { + names = append(names, manifest.Build.Artifacts[i].Name) + if manifest.Build.Artifacts[i].Name == name { + art = &manifest.Build.Artifacts[i] + } + } + } + if art == nil { + return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", ")) + } + if art.Kind != "bundle" || art.Language != "go" { + return nil, fmt.Errorf("the artifact %s of %s is a %s of language %q: this module builds Go bundles", name, manifest.Module, art.Kind, art.Language) + } + if strings.TrimSpace(art.From) == "" { + return nil, fmt.Errorf("the artifact %s of %s says nothing it is built from", name, manifest.Module) + } + + // Its own output directory, made empty: a build of the same tree twice is one build. + out := filepath.Join(dir, outDir, art.Name) + if err := os.RemoveAll(out); err != nil { + return nil, err + } + if err := os.MkdirAll(out, 0o755); err != nil { + return nil, fmt.Errorf("making the output directory for %s: %w", art.Name, err) + } + binary := binaryName(*art) + // One -ldflags, composed: a repeated flag is not a merged one (the build agent's issue 161). + linker := recipeLinkerFlags + if system := strings.TrimSpace(art.System); system != "" { + linker += " -X " + recipeSystemStamp + "=" + system + } + cmd := []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(art.From, "./")} + if r := m.Run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("go build of %s: %s", art.Name, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + } + archive, digest, err := buildstep.Pack(out) + if err != nil { + return nil, fmt.Errorf("packing %s: %w", art.Name, err) + } + artifacts := filepath.Join(m.Work, workspace, "artifacts") + if err := os.MkdirAll(artifacts, 0o700); err != nil { + return nil, err + } + path := filepath.Join(artifacts, strings.ReplaceAll(module, "/", "_")+"."+art.Name+".tar.gz") + if err := os.WriteFile(path, archive, 0o600); err != nil { + return nil, err + } + return BuildResult{Artifact: art.Name, Binary: binary, Digest: digest, Size: len(archive), Path: path}, nil +} + +// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed toolchain +// alone, the modules the fetch put in the module's cache and no way to fetch another (GOPROXY=off), so a build never +// reaches the network. A directory that vendors is built from its vendor directory. +func (m *Module) buildEnv(dir string) []string { + mod := "-mod=mod" + if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil { + mod = "-mod=vendor" + } + return []string{ + "CGO_ENABLED=0", + "GOFLAGS=" + recipeFlags + " " + mod, + "GOTOOLCHAIN=local", + "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), + "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), + "GOPROXY=off", + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build_test.go b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go new file mode 100644 index 00000000..181f7d87 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go @@ -0,0 +1,217 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 1487: the build step, with the compiler faked at the process boundary: it writes the file `go build -o` names. + +func manifestWith(artifact string) string { + return `{"module":"demo","version":"1","build":{"artifacts":[` + artifact + `]}}` +} + +const goBundle = `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","binary":"demo-bin","loads":["demo-bin"]}` + +// buildModule is a module with a workspace holding a module "modules/demo" with this manifest; the compiler writes +// content to the file go build is told. +func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates, *[]cmdRecord) { + t.Helper() + calls := &[]cmdRecord{} + states := newStates() + m := moduleWith(states, "go1.27.1") + m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir() + dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + *calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")}) + for i, a := range args { + if a == "-o" { + if err := os.WriteFile(args[i+1], []byte(content), 0o755); err != nil { + return forgekey.Ran{Status: 1, Stderr: err.Error()} + } + } + } + return forgekey.Ran{} + } + return m, states, calls +} + +func runBuild(t *testing.T, m *Module, states *keptStates, artifact string) (BuildResult, delivery.Step) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_build" { + if _, err := tool.Run(map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": artifact}); err != nil { + t.Fatal(err) + } + } + } + st := states.waitEnd(t) + var res BuildResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + } + return res, st +} + +// goEntry reads what the build agent's recipe says out of lines 155-191 of mesh-control's internal/builder/toolchain.go +// (testdata/captured/toolchain-go-entry.txt, copied from origin/main 073ccc29): the environment words and flags of the +// compile, the linker flags and the system stamp. +func goEntry(t *testing.T) (compile []string, linker []string, stamp string) { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "captured", "toolchain-go-entry.txt")) + if err != nil { + t.Fatal(err) + } + text := string(raw) + find := func(pattern string) string { + m := regexp.MustCompile(pattern).FindStringSubmatch(text) + if m == nil { + t.Fatalf("the copied recipe has nothing like %s", pattern) + } + return m[1] + } + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`Compile: \[\]string\{([^}]*)\}`), -1) { + compile = append(compile, q[1]) + } + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`LinkerFlags: \[\]string\{([^}]*)\}`), -1) { + linker = append(linker, q[1]) + } + return compile, linker, find(`SystemStamp: "([^"]+)"`) +} + +// 1487/AC1: the build command carries today's flags and environment: the copied recipe says env CGO_ENABLED=0 +// GOFLAGS=-trimpath -buildvcs=false, go build, -ldflags -s -w -X main.builtFor=, -o /, the package. +func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) { + m, states, calls := buildModule(t, manifestWith(goBundle), "binary") + res, st := runBuild(t, m, states, "tools") + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + compile, linker, stamp := goEntry(t) + // compile is: env CGO_ENABLED=0 "GOFLAGS=-trimpath -buildvcs=false" go build + if len(compile) != 5 || compile[0] != "env" || compile[3] != "go" || compile[4] != "build" { + t.Fatalf("the copied recipe's compile is %v", compile) + } + var c cmdRecord + for _, r := range *calls { + if strings.HasPrefix(r.line, "go ") { + c = r + } + } + env := strings.Join(c.env, "\n") + for _, want := range []string{compile[1], "GOFLAGS=" + strings.TrimPrefix(compile[2], "GOFLAGS=") + " -mod=mod", "GOTOOLCHAIN=local", "GOPROXY=off", + "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild")} { + if !strings.Contains(env, want) { + t.Errorf("the environment lacks %q:\n%s", want, env) + } + } + dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo") + out := filepath.Join(dir, ".mesh-build", "tools") + want := "go -C " + dir + " build -ldflags " + strings.Join(linker, " ") + " -X " + stamp + "=arch -o " + filepath.Join(out, "demo-bin") + " ./cmd/demo-tools" + if c.line != want { + t.Errorf("command = %q\nwant %q", c.line, want) + } + if res.Artifact != "tools" || res.Binary != "demo-bin" || !strings.HasPrefix(res.Digest, "sha256:") || res.Size == 0 { + t.Errorf("result = %+v", res) + } +} + +// The archive is the file the digest names, holding one executable of the name binaryName says; two builds of one tree +// are one digest. +func TestTheArchiveHoldsOneExecutableAndTwoBuildsAreOneDigest(t *testing.T) { + m, states, _ := buildModule(t, manifestWith(goBundle), "the program") + first, _ := runBuild(t, m, states, "tools") + second, _ := runBuild(t, m, states, "tools") + if first.Digest != second.Digest || first.Path != second.Path { + t.Errorf("two builds: %+v and %+v", first, second) + } + raw, err := os.ReadFile(first.Path) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(raw) + if first.Digest != "sha256:"+hex.EncodeToString(sum[:]) { + t.Errorf("the digest %s is not the archive's", first.Digest) + } + zr, err := gzip.NewReader(strings.NewReader(string(raw))) + if err != nil { + t.Fatal(err) + } + tr := tar.NewReader(zr) + h, err := tr.Next() + if err != nil || h.Name != "demo-bin" || h.Mode != 0o755 { + t.Fatalf("first entry = %+v, %v", h, err) + } + if body, _ := io.ReadAll(tr); string(body) != "the program" { + t.Errorf("the executable holds %q", body) + } + if _, err := tr.Next(); err != io.EOF { + t.Errorf("the archive holds more than one file") + } +} + +// 1487/AC3: an artifact of another language, or another kind, fails the step naming it. +func TestAnArtifactOfAnotherLanguageFailsTheStepNamingIt(t *testing.T) { + for artifact, want := range map[string]string{ + `{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"],"loads":["index.js"]}`: `language "typescript"`, + `{"name":"tools","kind":"archive","from":"files"}`: "archive", + } { + m, states, calls := buildModule(t, manifestWith(artifact), "x") + _, st := runBuild(t, m, states, "tools") + if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, want) || len(*calls) != 0 { + t.Errorf("%s: the step ended %s: %q after %d commands", artifact, st.State, st.Reason, len(*calls)) + } + } + m, states, _ := buildModule(t, manifestWith(goBundle), "x") + if _, st := runBuild(t, m, states, "nope"); st.State != delivery.StepFailed || !strings.Contains(st.Reason, `"nope"`) || !strings.Contains(st.Reason, "tools") { + t.Errorf("an unknown artifact: %s %q", st.State, st.Reason) + } +} + +// The executable is named as the build agent names it: the artifact's binary, else the package's directory, else the +// artifact. +func TestTheBinaryIsNamedAsTheBuildAgentNamesIt(t *testing.T) { + for artifact, want := range map[string]string{ + goBundle: "demo-bin", + `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","loads":["demo-tools"]}`: "demo-tools", + `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":".","loads":["tools"]}`: "tools", + } { + m, states, _ := buildModule(t, manifestWith(artifact), "x") + if res, st := runBuild(t, m, states, "tools"); res.Binary != want { + t.Errorf("%s: binary %q, want %q (%s %s)", artifact, res.Binary, want, st.State, st.Reason) + } + } +} + +func TestAModuleDirectoryOutsideTheWorkspaceIsRefused(t *testing.T) { + m, states, calls := buildModule(t, manifestWith(goBundle), "x") + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_build" { + tool.Run(map[string]any{"workspace": "ws1", "module": "../../..", "artifact": "tools"}) + } + } + if st := states.waitEnd(t); st.State != delivery.StepFailed || len(*calls) != 0 { + t.Errorf("the step ended %s after %d commands", st.State, len(*calls)) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/bus.go b/modules/mesh-build-go/cmd/mesh-build-go/bus.go new file mode 100644 index 00000000..8aa52548 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/bus.go @@ -0,0 +1,116 @@ +package main + +// The integration level's throwaway bus (novox/hq ADR 0470 §2; issue 1486): the module's own program runs a +// nats-server from the Go library, at the one version this module's go.mod names, with JetStream on, its store in the +// run's directory and a loopback port of its own. The program is this binary started again as `mesh-build-go bus +// `, so the bus is built from the same go.mod as the version the recipe names (ADR 0470 §4) and no second program +// has to be built and shipped; the tools bundle never serves a bus, and a bus never serves tools. It prints its +// address on one line and stops when its standard input closes. + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/nats-io/nats-server/v2/server" +) + +// busStartBound is how long a bus is given to answer its address. +const busStartBound = 30 * time.Second + +// busArg is the first argument that makes this binary a bus. +const busArg = "bus" + +// runBus is the bus program: args are the directory of its store. It returns the exit status. +func runBus(args []string) int { + if len(args) != 1 { + fmt.Fprintln(os.Stderr, "usage: mesh-build-go bus ") + return 2 + } + s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: args[0], NoLog: true, NoSigs: true}) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + go s.Start() + if !s.ReadyForConnections(busStartBound) { + fmt.Fprintln(os.Stderr, "the bus was not ready within", busStartBound) + s.Shutdown() + return 1 + } + fmt.Println(s.ClientURL()) + done := make(chan struct{}) + go func() { _, _ = io.Copy(io.Discard, os.Stdin); close(done) }() + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT) + select { + case <-done: + case <-sig: + } + s.Shutdown() + s.WaitForShutdown() + return 0 +} + +// startBus starts the bus of a run as a process of this binary under the step's environment, and answers its address. +func (m *Module) startBus(ctx context.Context, dir string) (string, func(), error) { + if m.StartBus != nil { + return m.StartBus(ctx, dir) + } + exe, err := os.Executable() + if err != nil { + return "", nil, err + } + cmd := exec.Command(exe, busArg, dir) + cmd.Env = baseEnv(m.Home) + in, err := cmd.StdinPipe() + if err != nil { + return "", nil, err + } + out, err := cmd.StdoutPipe() + if err != nil { + return "", nil, err + } + var errb strings.Builder + cmd.Stderr = &errb + if err := cmd.Start(); err != nil { + return "", nil, err + } + stop := func() { + _ = in.Close() + exited := make(chan struct{}) + go func() { _ = cmd.Wait(); close(exited) }() + select { + case <-exited: + case <-time.After(10 * time.Second): + _ = cmd.Process.Kill() + <-exited + } + } + line := make(chan string, 1) + go func() { + l, _ := bufio.NewReader(out).ReadString('\n') + line <- strings.TrimSpace(l) + }() + select { + case url := <-line: + if !strings.HasPrefix(url, "nats://") { + stop() + return "", nil, fmt.Errorf("the bus did not answer an address: %s", firstLine(errb.String()+" "+url)) + } + return url, stop, nil + case <-time.After(busStartBound + 5*time.Second): + stop() + return "", nil, fmt.Errorf("the bus did not answer an address within %s", busStartBound) + case <-ctx.Done(): + stop() + return "", nil, ctx.Err() + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go new file mode 100644 index 00000000..0adff4f4 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -0,0 +1,278 @@ +package main + +// The digest step (novox/hq ADR 0445 §5, ADR 0444; issue 1291): for each package of a workspace that has tests at a +// level, one digest of everything those tests read. The checker keys the verdict it keeps by it, so a package whose +// digest is unchanged is not run again. The tool answers in its call (it is short, ADR 0469 §2). + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind. +const DigestVersion = "1" + +// goPackage is the part of `go list -json` a digest reads. +type goPackage struct { + ImportPath string + Dir string + ForTest string + Standard bool + DepOnly bool + Deps []string + Module *goModule + // The files a package is built and tested from. + GoFiles, CgoFiles, CFiles, CXXFiles, MFiles, HFiles, FFiles, SFiles, SwigFiles, SwigCXXFiles, SysoFiles []string + TestGoFiles, XTestGoFiles []string + EmbedFiles, TestEmbedFiles, XTestEmbedFiles []string +} + +type goModule struct { + Path, Version, Sum string + Main bool + Replace *goModule + Dir string + GoVersion string +} + +func (p *goPackage) files() []string { + var all []string + for _, l := range [][]string{p.GoFiles, p.CgoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles, + p.SwigCXXFiles, p.SysoFiles, p.TestGoFiles, p.XTestGoFiles, p.EmbedFiles, p.TestEmbedFiles, p.XTestEmbedFiles} { + all = append(all, l...) + } + sort.Strings(all) + return slicesCompact(all) +} + +func slicesCompact(s []string) []string { + out := s[:0:0] + for i, v := range s { + if i == 0 || v != s[i-1] { + out = append(out, v) + } + } + return out +} + +// hasTests says whether the package has test files at the level go list was asked for. +func (p *goPackage) hasTests() bool { return len(p.TestGoFiles)+len(p.XTestGoFiles) > 0 } + +// DigestResult is what a digest step answers: one digest per package that has tests at the level, and the packages +// that have none. +type DigestResult struct { + Level string `json:"level"` + Packages map[string]string `json:"packages"` + WithoutTests []string `json:"without_tests,omitempty"` +} + +func (m *Module) digestTool() StepTool { + return StepTool{ + Name: "build_go_digest", + Description: "One digest per package of a fetched workspace, for one level, of everything that package's tests read: " + + "the files of each package the tests reach in the module's own tree, the version and checksum of each module " + + "outside it, the package's test data and embedded files, the Go version, the build tags and, at the " + + "integration level, the postgres major and nats-server versions. Answers in the call.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, + "level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"}, + }, + "required": []string{"workspace", "level"}, + }, + Run: m.digest, + } +} + +func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) { + workspace, _ := args["workspace"].(string) + level, _ := args["level"].(string) + if !workspaceOK.MatchString(workspace) { + return nil, fmt.Errorf("%q is not a workspace id", workspace) + } + tree := filepath.Join(m.Work, workspace, "tree") + if _, err := os.Stat(tree); err != nil { + return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) + } + var tags []string + header := []string{"digest " + DigestVersion, "level " + level, "go " + m.Recipe.Go} + switch level { + case LevelUnit: + case LevelIntegration: + if _, err := m.checkPostgres(ctx); err != nil { + return nil, err + } + tags = []string{"integration"} + header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer) + default: + return nil, fmt.Errorf("the level %q is neither unit nor integration", level) + } + header = append(header, "tags "+strings.Join(tags, ",")) + + dirs, err := m.goModDirs(ctx, tree) + if err != nil { + return nil, err + } + res := DigestResult{Level: level, Packages: map[string]string{}} + for _, d := range dirs { + pkgs, err := m.listPackages(ctx, tree, d, tags) + if err != nil { + return nil, err + } + if err := digestModule(&res, pkgs, tree, header); err != nil { + return nil, err + } + } + sort.Strings(res.WithoutTests) + return res, nil +} + +// listPackages is `go list -test -deps -json` of the directory's packages, by import path. +func (m *Module) listPackages(ctx context.Context, tree, dir string, tags []string) (map[string]*goPackage, error) { + abs := filepath.Join(tree, dir) + args := []string{"-C", abs, "list", "-test", "-deps", "-json"} + if len(tags) > 0 { + args = append(args, "-tags", strings.Join(tags, ",")) + } + args = append(args, "./...") + r := m.Run(ctx, m.goEnvFor(abs), "go", args...) + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("go list in %s: %s", dir, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) + } + return parsePackages(r.Stdout) +} + +// parsePackages reads `go list -json`'s stream of objects. +func parsePackages(out string) (map[string]*goPackage, error) { + pkgs := map[string]*goPackage{} + dec := json.NewDecoder(strings.NewReader(out)) + for { + var p goPackage + if err := dec.Decode(&p); err == io.EOF { + return pkgs, nil + } else if err != nil { + return nil, fmt.Errorf("go list answered something that is not its JSON: %w", err) + } + pkgs[p.ImportPath] = &p + } +} + +// digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main +// module that is listed, not only a dependency, and not a variant of another. +func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header []string) error { + roots := []string{} + for path, p := range pkgs { + if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main { + roots = append(roots, path) + } + } + sort.Strings(roots) + for _, path := range roots { + root := pkgs[path] + if !root.hasTests() { + res.WithoutTests = append(res.WithoutTests, path) + continue + } + testMain := pkgs[path+".test"] + if testMain == nil { + return fmt.Errorf("go list gave no test binary for %s", path) + } + d, err := packageDigest(root, testMain, pkgs, tree, header) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + res.Packages[path] = d + } + return nil +} + +// packageDigest is the digest of one package: a hash over a sorted text of the header, the files of every package its +// tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of +// every module outside it, and the package's own test data. A package of the standard library is the Go version's. +func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header []string) (string, error) { + reached := map[string]*goPackage{root.ImportPath: root} + for _, dep := range testMain.Deps { + if p := pkgs[dep]; p != nil { + reached[dep] = p + } + } + lines := []string{} + for _, p := range reached { + if p.Standard || p.Module == nil { + continue + } + mod := p.Module + if mod.Replace != nil { + mod = mod.Replace + } + if !mod.Main && mod.Version != "" { + lines = append(lines, "module "+mod.Path+"@"+mod.Version+" "+mod.Sum) + continue + } + for _, f := range p.files() { + l, err := fileLine(tree, p, f) + if err != nil { + return "", err + } + lines = append(lines, l) + } + } + data := filepath.Join(root.Dir, "testdata") + if _, err := os.Stat(data); err == nil { + err := filepath.WalkDir(data, func(path string, e fs.DirEntry, err error) error { + if err != nil || e.IsDir() { + return err + } + l, err := contentLine(tree, root, path) + lines = append(lines, l) + return err + }) + if err != nil { + return "", err + } + } + sort.Strings(lines) + h := sha256.New() + for _, l := range append(append([]string{}, header...), lines...) { + h.Write([]byte(l + "\n")) + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func fileLine(tree string, p *goPackage, name string) (string, error) { + return contentLine(tree, p, filepath.Join(p.Dir, name)) +} + +// contentLine is "file ": the path is the file's in the tree (the import path and the name when it lies +// outside, as a local replacement does), so a digest does not depend on where the workspace is. +func contentLine(tree string, p *goPackage, path string) (string, error) { + label := p.ImportPath + "/" + filepath.Base(path) + if rel, err := filepath.Rel(tree, path); err == nil && !strings.HasPrefix(rel, "..") { + label = filepath.ToSlash(rel) + } + info, err := os.Lstat(path) + if err != nil { + return "", err + } + if info.Mode()&os.ModeSymlink != 0 { + to, _ := os.Readlink(path) + return "link " + label + " " + to, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(raw) + return "file " + label + " " + hex.EncodeToString(sum[:]), nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go new file mode 100644 index 00000000..60a4efe4 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -0,0 +1,195 @@ +package main + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 1291: the digests of a fixture module (testdata/repos/digest): a's tests import b and read a/testdata, c's +// tests read an embedded file, d is reached by no test. They run the real `go list` (it loads packages, it builds +// none), through the step runner; only the listing of the tree's go.mod files and postgres's version are answered. + +// digestModule is a module over a copy of the fixture, in a workspace of its own. +func digestFixture(t *testing.T) (*Module, string) { + t.Helper() + work := t.TempDir() + tree := filepath.Join(work, "ws1", "tree") + if err := os.MkdirAll(tree, 0o755); err != nil { + t.Fatal(err) + } + if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", "digest")+"/.", tree).CombinedOutput(); err != nil { + t.Fatalf("%v %s", err, out) + } + m := moduleWith(newStates(), "go1.27.1") + m.Home, m.Work, m.Cache = t.TempDir(), work, t.TempDir() + real := StepRunner(m.Home) + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + switch name { + case "git": + return forgekey.Ran{Stdout: "go.mod\n"} + case "postgres": + return forgekey.Ran{Stdout: "postgres (PostgreSQL) " + strconv.Itoa(m.Recipe.PostgresMajor) + ".1\n"} + } + return real(ctx, env, name, args...) + } + return m, tree +} + +func digests(t *testing.T, m *Module, level string) DigestResult { + t.Helper() + out, err := m.digest(context.Background(), map[string]any{"workspace": "ws1", "level": level}) + if err != nil { + t.Fatal(err) + } + return out.(DigestResult) +} + +func edit(t *testing.T, tree, name, text string) { + t.Helper() + if err := os.WriteFile(filepath.Join(tree, name), []byte(text), 0o644); err != nil { + t.Fatal(err) + } +} + +// 1291/AC1: a file of b changes: a's digest changes (its tests import b), c's does not. +func TestADependencysFileChangesTheDigestOfThePackageWhoseTestsImportIt(t *testing.T) { + t.Parallel() + m, tree := digestFixture(t) + before := digests(t, m, "unit") + for _, p := range []string{"fixture/digest/a", "fixture/digest/c"} { + if len(before.Packages[p]) != 64 { + t.Fatalf("no digest for %s: %+v", p, before) + } + } + edit(t, tree, "b/b.go", "package b\n\n// B is reached by a's tests.\nfunc B() int { return 2 + 0 }\n") + after := digests(t, m, "unit") + if after.Packages["fixture/digest/a"] == before.Packages["fixture/digest/a"] { + t.Errorf("a's digest did not change when b did") + } + if after.Packages["fixture/digest/c"] != before.Packages["fixture/digest/c"] { + t.Errorf("c's digest changed when b did") + } + if again := digests(t, m, "unit"); again.Packages["fixture/digest/a"] != after.Packages["fixture/digest/a"] { + t.Errorf("one tree has two digests") + } +} + +// 1291/AC2: a file under a's testdata/ changes: a's digest changes. An embedded file does the same for c. +func TestATestdataFileAndAnEmbeddedFileChangeTheDigest(t *testing.T) { + t.Parallel() + m, tree := digestFixture(t) + before := digests(t, m, "unit") + edit(t, tree, "a/testdata/x.txt", "y\n") + after := digests(t, m, "unit") + if after.Packages["fixture/digest/a"] == before.Packages["fixture/digest/a"] { + t.Errorf("a's digest did not change with its testdata") + } + if after.Packages["fixture/digest/c"] != before.Packages["fixture/digest/c"] { + t.Errorf("c's digest changed with a's testdata") + } + edit(t, tree, "c/note.txt", "another note\n") + if again := digests(t, m, "unit"); again.Packages["fixture/digest/c"] == after.Packages["fixture/digest/c"] { + t.Errorf("c's digest did not change with its embedded file") + } +} + +// 1291/AC3: one tree at the unit and the integration level: the digests differ; at the integration level a change of the +// recipe's postgres major (or of the nats-server version) changes it, at the unit level it does not. +func TestTheLevelTheStoreAndTheBusAreInTheDigest(t *testing.T) { + t.Parallel() + m, _ := digestFixture(t) + unit, integration := digests(t, m, "unit"), digests(t, m, "integration") + a := "fixture/digest/a" + if unit.Packages[a] == integration.Packages[a] { + t.Errorf("the unit and the integration digest of a are one") + } + m.Recipe.PostgresMajor++ + if got := digests(t, m, "integration"); got.Packages[a] == integration.Packages[a] { + t.Errorf("a new postgres major left the integration digest as it was") + } + if got := digests(t, m, "unit"); got.Packages[a] != unit.Packages[a] { + t.Errorf("a new postgres major moved the unit digest") + } + m.Recipe.PostgresMajor-- + m.Recipe.NatsServer = "9.9.9" + if got := digests(t, m, "integration"); got.Packages[a] == integration.Packages[a] { + t.Errorf("a new nats-server version left the integration digest as it was") + } + m.Recipe.NatsServer = "2.11.17" + m.Recipe.Go = "go1.27.2" + if got := digests(t, m, "unit"); got.Packages[a] == unit.Packages[a] { + t.Errorf("a new Go version left the unit digest as it was") + } +} + +// 1291/AC4: a change in a package no test of a reaches leaves a's digest as it was; d, with no tests, has none. +func TestAPackageNoTestReachesChangesNoDigest(t *testing.T) { + t.Parallel() + m, tree := digestFixture(t) + before := digests(t, m, "unit") + edit(t, tree, "d/d.go", "package d\n\n// D is reached by no test.\nfunc D() int { return 5 }\n") + after := digests(t, m, "unit") + for p, d := range before.Packages { + if after.Packages[p] != d { + t.Errorf("the digest of %s changed with a package no test of it reaches", p) + } + } + if _, has := after.Packages["fixture/digest/d"]; has || strings.Join(after.WithoutTests, " ") != "fixture/digest/b fixture/digest/d" { + t.Errorf("without tests = %v, packages = %v", after.WithoutTests, after.Packages) + } +} + +// The digest does not depend on where the workspace lies. +func TestTheDigestIsTheSameInAnotherWorkspace(t *testing.T) { + t.Parallel() + m1, _ := digestFixture(t) + m2, _ := digestFixture(t) + if d1, d2 := digests(t, m1, "unit"), digests(t, m2, "unit"); d1.Packages["fixture/digest/a"] != d2.Packages["fixture/digest/a"] { + t.Errorf("one tree in two places has two digests") + } +} + +// A module outside the tree counts by its version and checksum, not by its files: the package of mesh-sdk, as go list +// gave it for this module's own dependency (testdata/captured/golist-external.json, which README describes). +func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("testdata", "captured", "golist-external.json")) + if err != nil { + t.Fatal(err) + } + pkgs, err := parsePackages(string(raw)) + if err != nil { + t.Fatal(err) + } + ext := pkgs["git.novox.be/novox/mesh-sdk/go/buildstep"] + if ext == nil || ext.Module == nil || ext.Module.Sum == "" || ext.Module.Version == "" || ext.Module.Main { + t.Fatalf("the capture holds no external package with a version and a checksum: %+v", ext) + } + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "r.go"), []byte("package r\n"), 0o644) + root := &goPackage{ImportPath: "x/r", Dir: dir, Module: &goModule{Path: "x", Main: true}, GoFiles: []string{"r.go"}, TestGoFiles: []string{"r.go"}} + testMain := &goPackage{ImportPath: "x/r.test", Deps: []string{ext.ImportPath}} + pkgs[root.ImportPath] = root + digest := func() string { + d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}) + if err != nil { + t.Fatal(err) + } + return d + } + d1 := digest() + ext.Module.Sum = "h1:changed=" + if d2 := digest(); d1 == d2 { + t.Errorf("a new checksum of a module outside the tree left the digest as it was") + } + ext.Module.Sum, ext.Module.Version = "h1:changed=", "v9.9.9" + if d3 := digest(); d3 == d1 { + t.Errorf("a new version left the digest as it was") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/emit_test.go b/modules/mesh-build-go/cmd/mesh-build-go/emit_test.go new file mode 100644 index 00000000..71fb11d5 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/emit_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +// Issues 1483 and 1488 (for the indexer, issue 1444): the guard emits toolchain.reported after each write of the +// toolchains key, with the node and the language. +func TestTheGuardEmitsToolchainReportedAfterEachWrite(t *testing.T) { + m := moduleWith(newStates(), "go1.27.1") + type event struct { + typ string + body map[string]string + } + var got []event + m.Emit = func(typ string, body any) error { + raw, _ := json.Marshal(body) + var b map[string]string + json.Unmarshal(raw, &b) + got = append(got, event{typ, b}) + return nil + } + if err := m.Check(context.Background()); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].typ != "toolchain.reported" || got[0].body["node"] != "ace" || got[0].body["language"] != "go" || len(got[0].body) != 2 { + t.Errorf("events = %+v, want one toolchain.reported for ace, go", got) + } + m.Check(context.Background()) + if len(got) != 2 { + t.Errorf("a second guard pass emitted %d events in all, want 2", len(got)) + } +} + +// The event is carried by the module's stream, not named in emits (ADR 0363: one place); the validator checks that. +func TestTheManifestDeclaresTheToolchainsStream(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Streams []struct { + Name string `json:"name"` + Events []string `json:"events"` + } `json:"streams"` + } + if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) != 1 || m.Streams[0].Name != "toolchains-go" || !slices.Equal(m.Streams[0].Events, []string{"toolchain.**"}) { + t.Errorf("streams = %+v (%v)", m.Streams, err) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go new file mode 100644 index 00000000..bc09c9b3 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -0,0 +1,174 @@ +package main + +// The fetch step (novox/hq ADR 0405 §5 §6, ADR 0466 §4 §5, ADR 0469 §2; issue 1485): a commit of a repository of +// the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and the Go modules every +// go.mod in it names, downloaded into the module's cache and verified against go.sum. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the +// remote of a fetch is ssh:////.git. +const ForgeAlias = "mesh-forge" + +// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it +// fetches and every round it checks its toolchain. A workspace lies in the module's work directory, never in its +// home (ADR 0466 §1). +const WorkspaceAge = 24 * time.Hour + +var ( + repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) + workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`) +) + +// FetchResult is what a passed fetch step says: the workspace it made, the commit in it and the directories whose +// Go modules it downloaded. +type FetchResult struct { + Workspace string `json:"workspace"` + Commit string `json:"commit"` + Modules []string `json:"modules"` +} + +// fetchTool is the fetch step tool. +func (m *Module) fetchTool() StepTool { + return StepTool{ + Name: "build_go_fetch", + Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " + + "workspace of its own, and download and verify the Go modules every go.mod in it names. Answers a step id at " + + "once; the step ends on its key in the steps state with the workspace's id.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "/ on the forge"}, + "commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"}, + "ref": map[string]any{"type": "string", "description": "refs/pull//head when only a pull request's head carries the commit"}, + }, + "required": []string{"repository", "commit"}, + }, + Long: true, + Run: m.fetch, + } +} + +func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { + repository, _ := args["repository"].(string) + commit, _ := args["commit"].(string) + ref, _ := args["ref"].(string) + if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") { + return nil, fmt.Errorf("%q is not a repository, /", repository) + } + m.CleanWorkspaces(m.now()) + id := buildstep.NewID() + root := filepath.Join(m.Work, id) + tree := filepath.Join(root, "tree") + if err := os.MkdirAll(tree, 0o700); err != nil { + return nil, err + } + remote := "ssh://" + ForgeAlias + "/" + repository + ".git" + if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref); err != nil { + return nil, err + } + dirs, err := m.goModDirs(ctx, tree) + if err != nil { + return nil, err + } + for _, d := range dirs { + if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil { + continue // a module that vendors resolves nothing + } + for _, verb := range []string{"download", "verify"} { + r := m.Run(ctx, m.goEnv(), "go", "-C", filepath.Join(tree, d), "mod", verb) + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("go mod %s in %s: %s", verb, d, firstLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr)) + } + } + } + return FetchResult{Workspace: id, Commit: commit, Modules: dirs}, nil +} + +// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order. +func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) { + r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod") + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr)) + } + dirs := []string{} + for _, l := range strings.Split(r.Stdout, "\n") { + if l = strings.TrimSpace(l); l != "" { + dirs = append(dirs, filepath.ToSlash(filepath.Dir(l))) + } + } + sort.Strings(dirs) + return dirs, nil +} + +// goEnv is what a go command of a step is given beyond the base environment: the installed toolchain and no other, +// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by +// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory +// that vendors, see goEnvFor. +func (m *Module) goEnv() []string { + return []string{ + "GOTOOLCHAIN=local", + "GOFLAGS=-mod=mod", + "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), + "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), + "GOPRIVATE=git.novox.be", + "GIT_TERMINAL_PROMPT=0", + "GIT_SSH_COMMAND=ssh -o BatchMode=yes", + } +} + +// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor +// directory, as fetch downloaded nothing for it, and every other by its go.mod and go.sum. +func (m *Module) goEnvFor(dir string) []string { + env := m.goEnv() + if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil { + env = append(env, "GOFLAGS=-mod=vendor") // the last value of a name is the one used + } + return env +} + +// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory. +func (m *Module) CleanWorkspaces(now time.Time) { + entries, err := os.ReadDir(m.Work) + if err != nil { + return + } + for _, e := range entries { + info, err := e.Info() + if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge { + continue + } + if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil { + m.Say("the workspace %s could not be removed: %v", e.Name(), err) + } + } +} + +// clock is the time now: Now when a test set it, else the wall clock. +func (m *Module) clock() time.Time { + if m.Now != nil { + return m.Now() + } + return time.Now() +} +func (m *Module) now() time.Time { return m.clock() } + +func firstLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + return l + } + } + return "" +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go new file mode 100644 index 00000000..d223cdaa --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -0,0 +1,292 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +const commitSHA = "0123456789abcdef0123456789abcdef01234567" + +// cmdRecord is one command a step ran. +type cmdRecord struct { + env []string + line string +} + +// fakeStep answers the commands of a fetch: git succeeds, rev-parse answers the commit, ls-files answers the +// go.mod files, and a go command answers by rule. +type fakeStep struct { + mu sync.Mutex + calls []cmdRecord + gomods string + gate chan struct{} // when set, the first git command waits for it + goRule func(line string) forgekey.Ran + vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout +} + +func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + line := name + " " + strings.Join(args, " ") + f.mu.Lock() + f.calls = append(f.calls, cmdRecord{env: env, line: line}) + f.mu.Unlock() + if f.gate != nil && strings.Contains(line, " init ") { + <-f.gate + } + switch { + case name == "git" && strings.Contains(line, " rev-parse HEAD"): + return forgekey.Ran{Stdout: commitSHA + "\n"} + case name == "git" && strings.Contains(line, " ls-files "): + return forgekey.Ran{Stdout: f.gomods} + case name == "git" && strings.Contains(line, " checkout "): + for i, a := range args { + if a == "-C" { + for _, v := range f.vendors { + os.MkdirAll(filepath.Join(args[i+1], v, "vendor"), 0o755) + os.WriteFile(filepath.Join(args[i+1], v, "vendor", "modules.txt"), nil, 0o644) + } + } + } + case name == "go" && f.goRule != nil: + return f.goRule(line) + } + return forgekey.Ran{} +} + +func (f *fakeStep) lines() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := []string{} + for _, c := range f.calls { + out = append(out, c.line) + } + return out +} + +func fetchModule(t *testing.T, f *fakeStep) (*Module, *keptStates) { + t.Helper() + states := newStates() + m := moduleWith(states, "go1.27.1") + m.Run, m.Home, m.Work, m.Cache = f.run, t.TempDir(), t.TempDir(), t.TempDir() + return m, states +} + +func fetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_fetch" { + return tool.Run + } + } + t.Fatal("no tool build_go_fetch") + return nil +} + +var fetchArgs = map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA} + +// 1485/AC1: git is asked through FetchCommit with the ssh remote, and go mod download and verify run in each go.mod +// directory (not in one that vendors) with GOTOOLCHAIN=local, GOFLAGS=-mod=mod and the module's cache paths. +func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) { + f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}} + m, states := fetchModule(t, f) + if _, err := fetchTool(t, m)(fetchArgs); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + lines := strings.Join(f.lines(), "\n") + for _, want := range []string{ + "remote add origin ssh://mesh-forge/novox/mesh-sdk.git", + "fetch --no-tags --depth 1 origin " + commitSHA, + "checkout -q --detach " + commitSHA, + } { + if !strings.Contains(lines, want) { + t.Errorf("git was not asked %q:\n%s", want, lines) + } + } + downloads, verifies := 0, 0 + for _, c := range f.calls { + switch { + case strings.HasPrefix(c.line, "go -C ") && strings.HasSuffix(c.line, " mod download"): + downloads++ + case strings.HasPrefix(c.line, "go -C ") && strings.HasSuffix(c.line, " mod verify"): + verifies++ + default: + continue + } + if strings.Contains(c.line, "vend") { + t.Errorf("a module that vendors was downloaded: %s", c.line) + } + env := strings.Join(c.env, " ") + for _, want := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be"} { + if !strings.Contains(env, want) { + t.Errorf("%s: environment lacks %s: %s", c.line, want, env) + } + } + } + if downloads != 2 || verifies != 2 { + t.Errorf("%d downloads and %d verifies, want 2 each (go.mod and sub/dir/go.mod):\n%s", downloads, verifies, lines) + } + var res FetchResult + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + if res.Commit != commitSHA || strings.Join(res.Modules, ",") != ".,sub/dir,vend" || res.Workspace == "" { + t.Errorf("result = %s", st.Result) + } + if _, err := os.Stat(filepath.Join(m.Work, res.Workspace, "tree")); err != nil { + t.Errorf("the workspace is not in the work directory: %v", err) + } +} + +// 1485/AC2: a variable such as MESH_REGISTRY_PASSWORD or a *_TOKEN of the process is seen by no command of the step: +// the runner builds the environment from scratch, and the step passes it nothing of the kind. +func TestNoCommandOfTheStepSeesAPasswordOrAToken(t *testing.T) { + t.Setenv("MESH_REGISTRY_PASSWORD", "hunter2-registry") + t.Setenv("FORGE_TOKEN", "hunter2-token") + t.Setenv("HOME", "/not/the/account") + r := StepRunner("/home/account") + got := r(context.Background(), []string{"EXTRA=1"}, "env") + if got.Status != 0 { + t.Fatalf("env: %+v", got) + } + for _, secret := range []string{"hunter2", "MESH_REGISTRY_PASSWORD", "FORGE_TOKEN", "/not/the/account"} { + if strings.Contains(got.Stdout, secret) { + t.Errorf("a command saw %q:\n%s", secret, got.Stdout) + } + } + for _, want := range []string{"HOME=/home/account", "EXTRA=1", "PATH="} { + if !strings.Contains(got.Stdout, want) { + t.Errorf("a command lacks %q:\n%s", want, got.Stdout) + } + } + + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + fetchTool(t, m)(fetchArgs) + states.waitEnd(t) + for _, c := range f.calls { + for _, w := range c.env { + if k, _, _ := strings.Cut(w, "="); strings.HasSuffix(k, "TOKEN") || strings.HasSuffix(k, "PASSWORD") || strings.HasSuffix(k, "SECRET") { + t.Errorf("%s was given %s", c.line, w) + } + } + } +} + +// 1485/AC3: go mod verify failing ends the step failed naming the directory and the first line of its output. +func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) { + f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", goRule: func(line string) forgekey.Ran { + if strings.HasSuffix(line, "tools mod verify") || strings.Contains(line, "/tools mod verify") { + return forgekey.Ran{Status: 1, Stdout: "\nexample.org/x v1.0.0: dir has been modified\nsecond line\n"} + } + return forgekey.Ran{} + }} + m, states := fetchModule(t, f) + if _, err := fetchTool(t, m)(fetchArgs); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != delivery.StepFailed { + t.Fatalf("the step ended %s, want failed", st.State) + } + if !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, "example.org/x v1.0.0: dir has been modified") || strings.Contains(st.Reason, "second line") { + t.Errorf("reason = %q", st.Reason) + } +} + +// 1485/AC4: the call answers a step id before the fetch ends, and the key then ends passed with the workspace. +func TestTheCallAnswersAStepIdBeforeTheFetchEnds(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n", gate: make(chan struct{})} + m, states := fetchModule(t, f) + out, err := fetchTool(t, m)(fetchArgs) + if err != nil { + t.Fatal(err) + } + answer := out.(map[string]any) + id, _ := answer["step"].(string) + if id == "" || answer["key"] != "ace."+id { + t.Fatalf("answer = %v, want a step id and its key", answer) + } + select { + case <-states.ended: + t.Fatal("the step ended while git was held") + case <-time.After(50 * time.Millisecond): + } + states.mu.Lock() + running := states.steps["ace."+id] + states.mu.Unlock() + if len(running) != 1 || running[0].State != delivery.StepRunning { + t.Errorf("the key holds %+v, want one running step", running) + } + close(f.gate) + select { + case key := <-states.ended: + if key != "ace."+id { + t.Errorf("ended %s", key) + } + case <-time.After(3 * time.Second): + t.Fatal("the step never ended") + } + states.mu.Lock() + last := states.steps["ace."+id][len(states.steps["ace."+id])-1] + states.mu.Unlock() + var res FetchResult + if last.State != delivery.StepPassed || json.Unmarshal(last.Result, &res) != nil || res.Workspace == "" { + t.Errorf("the key ends %+v", last) + } +} + +func TestARepositoryThatIsNotOwnerSlashNameIsRefusedBeforeAnythingRuns(t *testing.T) { + for _, repo := range []string{"", "mesh-sdk", "../etc/passwd", "a/b/c", "a/..", "-o/x", "https://x/y"} { + f := &fakeStep{} + m, states := fetchModule(t, f) + fetchTool(t, m)(map[string]any{"repository": repo, "commit": commitSHA}) + st := states.waitEnd(t) + if st.State != delivery.StepFailed || len(f.lines()) != 0 { + t.Errorf("%q: the step ended %s after %d commands", repo, st.State, len(f.lines())) + } + } +} + +// Workspaces older than a day are removed; younger ones, and anything that is not a workspace, stay. +func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) { + m := &Module{Work: t.TempDir()} + for name, age := range map[string]time.Duration{"old1": 25 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} { + dir := filepath.Join(m.Work, name) + os.MkdirAll(filepath.Join(dir, "tree"), 0o755) + when := time.Now().Add(-age) + os.Chtimes(dir, when, when) + } + m.CleanWorkspaces(time.Now()) + for name, want := range map[string]bool{"old1": false, "young": true, "Keep-This": true} { + _, err := os.Stat(filepath.Join(m.Work, name)) + if (err == nil) != want { + t.Errorf("%s present = %v, want %v", name, err == nil, want) + } + } +} + +// The work and cache directories the code uses are the ones the manifest places, and neither is in the home. +func TestWorkAndCacheAreTheDirectoriesTheManifestPlacesOutsideTheHome(t *testing.T) { + m := readManifest(t) + home := resource(t, m, "account")["home"].(string) + for id, path := range map[string]string{"work": WorkDir, "cache": CacheDir} { + if resource(t, m, id)["path"] != path { + t.Errorf("%s: the manifest places %v, the code uses %s", id, resource(t, m, id)["path"], path) + } + if path == home || strings.HasPrefix(path, home+"/") { + t.Errorf("%s %s is in the home %s", id, path, home) + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go new file mode 100644 index 00000000..b4e1d4b1 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go @@ -0,0 +1,246 @@ +package main + +// The forge side of a build module (novox/hq ADR 0466 §2 and §4, issue 1484). This file is the same for every build +// module: mesh-build-node takes a copy and changes the three constants below. + +import ( + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +const ( + forgeModule = "mesh-build-go" + forgeHome = "/var/lib/mesh-build-go" + forgeModuleD = "../.." // the module's directory, where module.json and files/ are +) + +type forgeManifest struct { + Requires []any `json:"requires"` + State []any `json:"state"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` +} + +func readForgeManifest(t *testing.T) forgeManifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join(forgeModuleD, "module.json")) + if err != nil { + t.Fatal(err) + } + var m forgeManifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +func forgeTemplate(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(forgeModuleD, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +func forgeResource(t *testing.T, m forgeManifest, id string) map[string]any { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + if r["owner"] != forgeModule { + t.Errorf("%s is owned by %v, not the module's own account", id, r["owner"]) + } + if p, _ := r["path"].(string); !strings.HasPrefix(p, "${dir:") || strings.Contains(p, "account-home") { + t.Errorf("%s is at %q: a path of the account's own home (${dir:...}), never the operator's", id, p) + } + return r + } + } + t.Fatalf("no resource %q", id) + return nil +} + +var forgeSample = map[string]string{ + "known_host": "[novox.internal]:222 ssh-ed25519 AAAAC3Nza", + "host_key": "ssh-ed25519 AAAAC3Nza", + "ssh_host": "novox.internal", + "ssh_alias": "mesh-forge", + "ssh_port": "222", + "ssh_base": "ssh://git@novox.internal:222/", + "https_base": "https://git.novox.be/", +} + +// forgeRender fills ${bound:forge-account:} with the sample values, as the provisioner client does. +func forgeRender(t *testing.T, s string) string { + t.Helper() + for k, v := range forgeSample { + s = strings.ReplaceAll(s, "${bound:forge-account:"+k+"}", v) + } + if strings.Contains(s, "${") { + t.Errorf("unfilled placeholder in %q", s) + } + return s +} + +// 1484/AC1: given sample bound values, the ssh config has Host mesh-forge, IdentitiesOnly yes and the key under the +// module's own home; known_hosts carries the host key; git's config rewrites https_base to ssh://mesh-forge/; no +// ${ is left. +func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) { + m := readForgeManifest(t) + + cfg := forgeResource(t, m, "ssh-config") + want := "Host mesh-forge\n HostName novox.internal\n Port 222\n HostKeyAlias mesh-forge\n User git\n IdentityFile ~/.ssh/mesh_ed25519\n IdentitiesOnly yes\n" + if got := forgeRender(t, forgeTemplate(t, cfg)); got != want { + t.Errorf("ssh config = %q, want %q", got, want) + } + if cfg["path"] != "${dir:ssh-dir}/config" || cfg["mode"] != "0600" || cfg["into"] != nil { + t.Errorf("ssh config resource = %v, want the whole file ${dir:ssh-dir}/config at mode 0600", cfg) + } + + kh := forgeResource(t, m, "known-hosts") + if got := strings.TrimSpace(forgeRender(t, forgeTemplate(t, kh))); got != "mesh-forge ssh-ed25519 AAAAC3Nza" { + t.Errorf("known_hosts = %q", got) + } + + g := forgeResource(t, m, "git-config") + wantGit := "[url \"ssh://mesh-forge/\"]\n\tinsteadOf = https://git.novox.be/\n" + if got := forgeRender(t, forgeTemplate(t, g)); got != wantGit { + t.Errorf("git config = %q, want %q", got, wantGit) + } + + dir := forgeResource(t, m, "ssh-dir") + if dir["mode"] != "0700" || dir["path"] != "${dir:state}/.ssh" { + t.Errorf("ssh-dir = %v, want ${dir:state}/.ssh at mode 0700", dir) + } +} + +// 1484/AC2: the module requires forge-account, keeps the state keys, and every file resource takes its text from a +// template in the files/ archive (novox/hq ADR 0349 §2). +func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) { + m := readForgeManifest(t) + if len(m.Requires) != 1 || m.Requires[0] != "forge-account" { + t.Errorf("requires = %v, want [forge-account]", m.Requires) + } + keeps := false + for _, s := range m.State { + if s == "keys" { + keeps = true + } + } + if !keeps { + t.Errorf("state = %v lacks keys", m.State) + } + files := 0 + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline", r["id"]) + } + if r["type"] == "file" { + files++ + forgeTemplate(t, r) + } + } + if files == 0 { + t.Errorf("no file resource: the forge's ssh files are missing") + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" && a["name"] == "files" { + packs++ + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives named files, not one", packs) + } +} + +// forgeEd25519Line is a public key line with a well-formed ed25519 blob. +func forgeEd25519Line(comment string) string { + blob := []byte{} + for _, part := range [][]byte{[]byte("ssh-ed25519"), make([]byte, 32)} { + l := make([]byte, 4) + binary.BigEndian.PutUint32(l, uint32(len(part))) + blob = append(append(blob, l...), part...) + } + return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(blob) + " " + comment +} + +type forgeKeys struct{ puts map[string][]byte } + +func (f *forgeKeys) Put(key string, value any) (uint64, error) { + raw, _ := json.Marshal(value) + if f.puts == nil { + f.puts = map[string][]byte{} + } + f.puts[key] = raw + return uint64(len(f.puts)), nil +} + +// forgeKeygen stands in for ssh-keygen: it writes the pair the command names. +func forgeKeygen(_ context.Context, _ []string, name string, args ...string) forgekey.Ran { + var file, comment string + for i, a := range args { + switch a { + case "-f": + file = args[i+1] + case "-C": + comment = args[i+1] + } + } + if name != "ssh-keygen" || file == "" { + return forgekey.Ran{Status: 1, Stderr: "unexpected"} + } + os.WriteFile(file, []byte("PRIVATE\n"), 0o600) + os.WriteFile(file+".pub", []byte(forgeEd25519Line(comment)+"\n"), 0o644) + return forgekey.Ran{} +} + +// 1484/AC3: given the bundle started with an empty home, the keeper writes . with kind builder, and +// the key is in the home's .ssh. +func TestTheKeeperWritesTheModulesEntryWithKindBuilder(t *testing.T) { + home, st := t.TempDir(), &forgeKeys{} + if err := keeperFor(home, "ace", forgeModule, forgeKeygen, st).Once(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(home, ".ssh", "mesh_ed25519")); err != nil { + t.Errorf("no key in an empty home: %v", err) + } + raw, ok := st.puts["ace."+forgeModule] + if !ok || len(st.puts) != 1 { + t.Fatalf("entries = %v, want one at ace.%s", st.puts, forgeModule) + } + var e forgekey.KeyEntry + if err := json.Unmarshal(raw, &e); err != nil { + t.Fatal(err) + } + if e.Kind != "builder" || e.Node != "ace" || e.Account != forgeModule || !strings.HasPrefix(e.Public, "ssh-ed25519 ") { + t.Errorf("entry = %s", raw) + } + if strings.Contains(strings.ToUpper(string(raw)), "PRIVATE") { + t.Errorf("the entry carries private material: %s", raw) + } +} + +// The home the manifest places is the one the account is given, so the keeper's home and the templates' agree. +func TestTheAccountsHomeIsTheOneTheManifestNames(t *testing.T) { + m := readForgeManifest(t) + for _, r := range m.Resources { + if r["type"] == "user" && r["name"] == forgeModule && r["home"] != forgeHome { + t.Errorf("the account's home is %v, want %s", r["home"], forgeHome) + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go new file mode 100644 index 00000000..d96c4bb5 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -0,0 +1,77 @@ +package main + +// The integration level (novox/hq ADR 0445 §2 §3, ADR 0470; issue 1486): `go test -tags integration` over the +// workspace, against a throwaway postgres and a throwaway bus started for this run alone, handed to the tests as +// MESH_TEST_POSTGRES and MESH_TEST_NATS. Its setup (compiling, the store, the bus) and its running time are measured and +// judged apart, 30 s and 60 s; both servers are stopped and the run's directory removed whatever the tests did. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// The integration level's bounds (ADR 0445 §3). +const ( + IntegrationSetup = 30 * time.Second + IntegrationRun = 60 * time.Second +) + +func (m *Module) integrationBounds() Bounds { + b := m.IntegrationBounds + if b.Setup <= 0 { + b.Setup = IntegrationSetup + } + if b.Run <= 0 { + b.Run = IntegrationRun + } + return b +} + +func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string) (res *TestResult, err error) { + major, err := m.checkPostgres(ctx) + if err != nil { + return nil, err // a *buildstep.Refusal: this level only + } + run := filepath.Join(m.Work, workspace, "run-"+buildstep.NewID()) + if err := os.MkdirAll(run, 0o700); err != nil { + return nil, err + } + // The run's directory goes, and the servers stop, however the level ends: passed, failed, a panic. + defer os.RemoveAll(run) + + began := m.clock() + dirs, err := m.goModDirs(ctx, tree) + if err != nil { + return nil, err + } + postgres, stopStore, err := m.startStore(ctx, run) + if err != nil { + return nil, fmt.Errorf("the integration level's store: %w", err) + } + defer stopStore() + nats, stopBus, err := m.startBus(ctx, filepath.Join(run, "nats")) + if err != nil { + return nil, fmt.Errorf("the integration level's bus: %w", err) + } + defer stopBus() + env := []string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats} + // Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run + // below finds them built. Whether they compile is for the run to say; its failure is not read here. + for _, d := range dirs { + m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", "integration", "-run", "^$", "-count=1", "./...") + } + setup := m.clock().Sub(began) + + return m.runLevel(ctx, tree, levelOpts{ + level: LevelIntegration, tags: []string{"integration"}, packages: packages, bounds: m.integrationBounds(), + env: env, + setup: setup, wall: true, + store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer, + }) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go new file mode 100644 index 00000000..7f51597f --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go @@ -0,0 +1,366 @@ +package main + +import ( + "context" + "encoding/json" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 1486: the integration level, with the store and the bus faked at the process boundary: the commands the level +// runs are recorded, and a clock the commands advance stands in for the time they would take. + +type integFake struct { + mu sync.Mutex + calls []cmdRecord + clock time.Time + postgres string // what `postgres --version` prints + captured string // the go test -json output of the run + // advance is how long a phase takes on the fake clock: compile, start (of the store), test. + advance map[string]time.Duration + busURL string + busDir string + busDown bool + runDirs []string // run directories seen when the store was started, to be gone at the end + panicOn string // a phase that panics +} + +func newIntegFake(t *testing.T, captured string) *integFake { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "captured", captured+".json")) + if err != nil { + t.Fatal(err) + } + return &integFake{clock: time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC), postgres: "postgres (PostgreSQL) 18.6", + captured: string(raw), advance: map[string]time.Duration{}, busURL: "nats://127.0.0.1:45222"} +} + +func (f *integFake) now() time.Time { f.mu.Lock(); defer f.mu.Unlock(); return f.clock } + +func (f *integFake) tick(phase string) { + f.mu.Lock() + f.clock = f.clock.Add(f.advance[phase]) + f.mu.Unlock() + if f.panicOn == phase { + panic("the " + phase + " phase panicked") + } +} + +func (f *integFake) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + line := name + " " + strings.Join(args, " ") + f.mu.Lock() + f.calls = append(f.calls, cmdRecord{env: env, line: line}) + f.mu.Unlock() + switch { + case name == "postgres": + return forgekey.Ran{Stdout: f.postgres + "\n"} + case name == "git" && len(args) > 2 && args[2] == "ls-files": + return forgekey.Ran{Stdout: "go.mod\n"} + case name == "go" && strings.Contains(line, "-run ^$"): + f.tick("compile") + case name == "go": + f.tick("test") + status := 0 + if strings.Contains(f.captured, `"Action":"fail"`) { + status = 1 + } + return forgekey.Ran{Stdout: f.captured, Status: status} + case name == "pg_ctl" && strings.HasSuffix(line, " start"): + f.tick("start") + for i, a := range args { + if a == "-D" { + f.runDirs = append(f.runDirs, filepath.Dir(filepath.Dir(args[i+1]))) + } + } + } + return forgekey.Ran{} +} + +func (f *integFake) startBus(_ context.Context, dir string) (string, func(), error) { + f.mu.Lock() + f.busDir = dir + f.mu.Unlock() + return f.busURL, func() { f.mu.Lock(); f.busDown = true; f.mu.Unlock() }, nil +} + +func (f *integFake) lines() string { + f.mu.Lock() + defer f.mu.Unlock() + out := []string{} + for _, c := range f.calls { + out = append(out, c.line) + } + return strings.Join(out, "\n") +} + +func integModule(t *testing.T, f *integFake) (*Module, *keptStates) { + t.Helper() + states := newStates() + m := moduleWith(states, "go1.27.1") + m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir() + m.Run, m.Now, m.StartBus = f.run, f.now, f.startBus + if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil { + t.Fatal(err) + } + return m, states +} + +func runLevelTool(t *testing.T, m *Module, states *keptStates, level string) (TestResult, delivery.Step) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name != "build_go_test" { + continue + } + if _, err := tool.Run(map[string]any{"workspace": "ws1", "level": level}); err != nil { + t.Fatal(err) + } + } + st := states.waitEnd(t) + var res TestResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + } + return res, st +} + +// 1486/AC2: the store's start carries listen_addresses=” and a socket directory under the run; the tests are given +// its address and the bus's, and the run's directory is in the workspace's. +func TestTheStoreListensOnASocketUnderTheRunAndNowhereElse(t *testing.T) { + f := newIntegFake(t, "slow") + m, states := integModule(t, f) + res, st := runLevelTool(t, m, states, "integration") + if st.State != delivery.StepPassed || !res.Passed || res.Level != "integration" { + t.Fatalf("the step ended %s (%s): %+v", st.State, st.Reason, res) + } + var start, initdb string + for _, c := range f.calls { + if strings.HasPrefix(c.line, "pg_ctl ") && strings.HasSuffix(c.line, " start") { + start = c.line + } + if strings.HasPrefix(c.line, "initdb ") { + initdb = c.line + } + } + run := f.runDirs[0] + if !strings.HasPrefix(run, filepath.Join(m.Work, "ws1", "run-")) { + t.Errorf("the run directory is %s, not under the workspace's", run) + } + _, afterK, _ := strings.Cut(start, "-k ") + socket, _, _ := strings.Cut(afterK, " ") + if len(socket)+len(socketName) >= 100 { + t.Errorf("the socket %q is too long for postgres", socket) + } + for _, want := range []string{"listen_addresses=''", "-k " + socket, "fsync=off", "-w"} { + if !strings.Contains(start, want) { + t.Errorf("the store's start %q lacks %q", start, want) + } + } + if strings.Contains(start, "listen_addresses='*'") || strings.Contains(start, "-h ") || strings.Contains(start, "port=") { + t.Errorf("the store listens on TCP: %s", start) + } + if !strings.Contains(initdb, "-D "+filepath.Join(run, "pg", "data")) || !strings.Contains(initdb, "--auth=trust") { + t.Errorf("initdb = %q", initdb) + } + var test cmdRecord + for _, c := range f.calls { + if strings.HasPrefix(c.line, "go ") && strings.Contains(c.line, "-json") { + test = c + } + } + if !strings.Contains(test.line, "-tags integration") { + t.Errorf("the run does not ask for the tag: %s", test.line) + } + env := map[string]string{} + for _, w := range test.env { + k, v, _ := strings.Cut(w, "=") + env[k] = v + } + u, err := url.Parse(env["MESH_TEST_POSTGRES"]) + if err != nil || u.Scheme != "postgres" || u.Query().Get("host") != socket || u.Host != "" { + t.Errorf("MESH_TEST_POSTGRES = %q", env["MESH_TEST_POSTGRES"]) + } + if strings.Count(strings.TrimPrefix(env["MESH_TEST_POSTGRES"], "postgres://"), "/") != 1 { + t.Errorf("MESH_TEST_POSTGRES = %q has a slash beyond the one before the database: the controller's tests cut it at the last", env["MESH_TEST_POSTGRES"]) + } + if env["MESH_TEST_NATS"] != "nats://127.0.0.1:45222" || f.busDir != filepath.Join(run, "nats") { + t.Errorf("MESH_TEST_NATS = %q, bus dir %q", env["MESH_TEST_NATS"], f.busDir) + } + if res.Store != "postgres 18" || res.Bus != "nats-server "+m.Recipe.NatsServer { + t.Errorf("the result names %q and %q", res.Store, res.Bus) + } +} + +// 1486/AC3: a postgres of major 17 refuses the integration level, naming 17 and 18; the unit level still runs. +func TestAnotherPostgresMajorRefusesTheIntegrationLevelOnly(t *testing.T) { + f := newIntegFake(t, "slow") + f.postgres = "postgres (PostgreSQL) 17.2" + m, states := integModule(t, f) + _, st := runLevelTool(t, m, states, "integration") + if st.State != delivery.StepRefused || !strings.Contains(st.Reason, "17") || !strings.Contains(st.Reason, "18") { + t.Fatalf("the integration step ended %s: %q", st.State, st.Reason) + } + if strings.Contains(f.lines(), "initdb") || strings.Contains(f.lines(), "-tags") { + t.Errorf("the refused level ran something:\n%s", f.lines()) + } + res, st := runLevelTool(t, m, states, "unit") + if st.State != delivery.StepPassed || !res.Passed || res.Level != "unit" { + t.Errorf("the unit level: %s %+v", st.State, res) + } +} + +// 1486/AC4: a setup over thirty seconds fails the level naming the setup bound; a run over sixty names the running +// bound and the slowest tests. The clock the commands move stands in for the time they take. +func TestSetupAndRunningTimeAreJudgedApart(t *testing.T) { + f := newIntegFake(t, "slow") + f.advance["compile"], f.advance["start"] = 20*time.Second, 15*time.Second + m, states := integModule(t, f) + res, _ := runLevelTool(t, m, states, "integration") + if !res.Passed || !res.OverBound || res.OverWhich != "setup" || res.Setup != 35 || !strings.Contains(res.Why, "setup took 35.00 s, over its setup bound of 30 s") || strings.Contains(res.Why, "slowest tests") { + t.Errorf("a setup of 35 s: %+v", res) + } + + g := newIntegFake(t, "slow") + g.advance["test"] = 61 * time.Second + m, states = integModule(t, g) + res, _ = runLevelTool(t, m, states, "integration") + if !res.Passed || res.OverWhich != "run" || res.Elapsed != 61 || !strings.Contains(res.Why, "took 61.00 s, over its bound of 60 s; the slowest tests: fixture/slow TestTakesAWhile") || strings.Contains(res.Why, "setup") { + t.Errorf("a run of 61 s: %+v", res) + } + + h := newIntegFake(t, "slow") + h.advance["compile"], h.advance["test"] = 29*time.Second, 59*time.Second + m, states = integModule(t, h) + if res, _ = runLevelTool(t, m, states, "integration"); !res.Passed || res.OverBound || res.Setup != 29 || res.Elapsed != 59 { + t.Errorf("29 s of setup and 59 s of running are within both bounds: %+v", res) + } +} + +// 1486/AC5: a run that fails, and a run that panics: the store is stopped, the bus is stopped, the run's directory is +// gone. +func TestAFailedOrPanickedRunStopsTheServersAndRemovesTheRunDirectory(t *testing.T) { + f := newIntegFake(t, "failing") + m, states := integModule(t, f) + res, _ := runLevelTool(t, m, states, "integration") + if res.Passed { + t.Fatalf("the failing fixture passed: %+v", res) + } + checkCleaned := func(f *integFake, m *Module) { + t.Helper() + lines := f.lines() + if !strings.Contains(lines, "pg_ctl -D "+filepath.Join(f.runDirs[0], "pg", "data")+" -m immediate -w stop") { + t.Errorf("the store was not stopped:\n%s", lines) + } + if strings.LastIndex(lines, " stop") < strings.LastIndex(lines, "-json") { + t.Errorf("the store was stopped before the tests ended:\n%s", lines) + } + if !f.busDown { + t.Errorf("the bus was not stopped") + } + if _, err := os.Stat(f.runDirs[0]); !os.IsNotExist(err) { + t.Errorf("the run directory %s is still there (%v)", f.runDirs[0], err) + } + } + checkCleaned(f, m) + + g := newIntegFake(t, "failing") + g.panicOn = "test" + m, _ = integModule(t, g) + func() { + defer func() { recover() }() + m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil) + }() + checkCleaned(g, m) +} + +// A store that does not start ends the step failed, with the server stopped and the directory gone. +func TestAStoreThatDoesNotStartEndsTheStepFailedAndLeavesNothing(t *testing.T) { + f := newIntegFake(t, "slow") + m, states := integModule(t, f) + inner := m.Run + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + r := inner(ctx, env, name, args...) + if name == "pg_ctl" && strings.HasSuffix(strings.Join(args, " "), " start") { + return forgekey.Ran{Status: 1, Stderr: "pg_ctl: could not start server\n"} + } + return r + } + _, st := runLevelTool(t, m, states, "integration") + if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "pg_ctl: could not start server") { + t.Errorf("the step ended %s: %q", st.State, st.Reason) + } + if !strings.Contains(f.lines(), " stop") { + t.Errorf("the half-started store was not stopped:\n%s", f.lines()) + } + if f.busDir != "" { + t.Errorf("the bus was started after the store failed") + } + if entries, _ := os.ReadDir(filepath.Join(m.Work, "ws1")); len(entries) != 1 || entries[0].Name() != "tree" { + t.Errorf("the workspace holds %v, want only its tree", entries) + } +} + +func TestThePostgresMajorIsReadFromTheVersionLine(t *testing.T) { + for line, want := range map[string]int{"postgres (PostgreSQL) 18.6": 18, "postgres (PostgreSQL) 9.6.24": 9, "postgres (PostgreSQL) 17.2 (Arch)": 17} { + f := &integFake{postgres: line, advance: map[string]time.Duration{}} + m := &Module{Run: f.run} + if got, err := m.postgresMajor(context.Background()); err != nil || got != want { + t.Errorf("%q = %d, %v; want %d", line, got, err, want) + } + } + if _, err := (&Module{Run: (&integFake{postgres: "no", advance: map[string]time.Duration{}}).run}).postgresMajor(context.Background()); err == nil { + t.Errorf("accepted a line that is not postgres's") + } +} + +// Issue 1486 review: postgres refuses a socket path of 108 bytes and over. A run directory deep in the work directory +// gets a short directory of its own for the socket, removed with the store; a short one keeps it under the run. +func TestTheSocketPathStaysUnderTheLimitForALongRunDirectory(t *testing.T) { + long := filepath.Join(t.TempDir(), strings.Repeat("deep", 20), "ws1", "run-mvh5uc96142806") + if err := os.MkdirAll(long, 0o700); err != nil { + t.Fatal(err) + } + dir, short, err := socketDir(long) + if err != nil || !short || len(dir)+len(socketName) >= 100 { + t.Fatalf("socketDir(long) = %q, %v, %v", dir, short, err) + } + if _, err := os.Stat(dir); err != nil { + t.Errorf("the short directory was not made: %v", err) + } + os.RemoveAll(dir) + + near := t.TempDir() + if dir, short, err := socketDir(near); err != nil || (len(near) < 50 && (short || !strings.HasPrefix(dir, near))) { + t.Errorf("socketDir(short) = %q, %v, %v", dir, short, err) + } + + // Through the store: the short directory is removed when the store is stopped. + f := newIntegFake(t, "slow") + m, _ := integModule(t, f) + _, stop, err := m.startStore(context.Background(), long) + if err != nil { + t.Fatal(err) + } + var sock string + for _, c := range f.calls { + if _, a, ok := strings.Cut(c.line, "-k "); ok && strings.HasPrefix(c.line, "pg_ctl") { + sock, _, _ = strings.Cut(a, " ") + } + } + if sock == "" || len(sock)+len(socketName) >= 100 { + t.Fatalf("the store was started with the socket %q", sock) + } + stop() + if _, err := os.Stat(sock); !os.IsNotExist(err) { + t.Errorf("the short socket directory %s outlived the store (%v)", sock, err) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/keys.go b/modules/mesh-build-go/cmd/mesh-build-go/keys.go new file mode 100644 index 00000000..fb93a0de --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/keys.go @@ -0,0 +1,42 @@ +package main + +// The module's own forge key (novox/hq ADR 0466 §2, issue 1484): mesh-sdk's key keeper, kind builder, for the +// module's account in its home. It makes ~/.ssh/mesh_ed25519 when it is missing and writes one entry to this +// module's `keys` state at ., which gitea's provider reads to register the key on the forge user +// that reads and pushes nothing (ADR 0466 §3). + +import ( + "context" + "fmt" + "os" + "os/user" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// keeperFor is this module's keeper: go/forgekey's, of kind builder, for account in home on node, whose entry lies +// at . of the keys state. +func keeperFor(home, node, account string, run forgekey.Runner, state forgekey.KeyPutter) *forgekey.KeyKeeper { + return &forgekey.KeyKeeper{Home: home, Node: node, Account: account, Kind: forgekey.KindBuilder, Run: run, State: state, Now: time.Now} +} + +// startKeeper runs the keeper when the runtime launched this process for a node: outside a launch — a build, a +// check — there is no state to put the entry in. +func startKeeper(ctx context.Context) { + node := strings.TrimSpace(os.Getenv("MESH_NODE")) + u, err := user.Current() + if node == "" || err != nil || u.HomeDir == "" { + return + } + k := keeperFor(u.HomeDir, node, u.Username, forgekey.ExecRunner, keysState{}) + k.Keep(ctx, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-go] "+f+"\n", a...) }) +} + +type keysState struct{} + +func (keysState) Put(key string, value any) (uint64, error) { + return stdio.State("keys").Put(key, value) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/levels.go b/modules/mesh-build-go/cmd/mesh-build-go/levels.go new file mode 100644 index 00000000..9631252a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/levels.go @@ -0,0 +1,262 @@ +package main + +// The judging of a test level (novox/hq ADR 0445 §2 §3; issues 1290 and 1486): `go test -json` read into a result +// that says, per package, what ran and for how long, which tests failed and why, and whether the level kept its +// bound. A level that failed is still a finished step: the verdict is in the result, and a step that ends failed is a +// step that could not run. + +import ( + "bufio" + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// The two levels (ADR 0445 §2). +const ( + LevelUnit = "unit" + LevelIntegration = "integration" +) + +// UnitBound is the unit level's one bound: its running time, five seconds for the level, no setup (ADR 0445 §3). The +// level's time is that of its slowest package, as packages run side by side, the compile not counted. +const UnitBound = 5 * time.Second + +// Bounds are the bounds a level is judged by: Setup is the time of everything before the tests run (compiling, +// starting the store and the bus; the unit level has none) and Run the tests' own time. +type Bounds struct { + Setup, Run time.Duration +} + +// PackageResult is one package's part of a level. +type PackageResult struct { + Dir string `json:"dir"` + Package string `json:"package"` + Passed bool `json:"passed"` + Elapsed float64 `json:"elapsed"` + Tests int `json:"tests"` + // NoTests is a package with no test files (at this level: a file with the other level's build tag is none). + NoTests bool `json:"no_tests,omitempty"` +} + +// TestTime is one test and the time it took. +type TestTime struct { + Package string `json:"package"` + Test string `json:"test"` + Elapsed float64 `json:"elapsed"` +} + +// Failure is a test that failed, or a package that failed with no test to name (a build that failed, a TestMain that +// panicked); Output is the first lines it wrote. +type Failure struct { + Package string `json:"package"` + Test string `json:"test,omitempty"` + Output string `json:"output,omitempty"` +} + +// TestResult is what a test step passes with. +type TestResult struct { + Level string `json:"level"` + Passed bool `json:"passed"` + // Why says, in words, why the level failed, or that it has no tests. + Why string `json:"why,omitempty"` + NoTests bool `json:"no_tests,omitempty"` + Tests int `json:"tests"` + Skipped int `json:"skipped"` + Packages []PackageResult `json:"packages"` + // Elapsed is the level's running time in seconds: the slowest package's. + Elapsed float64 `json:"elapsed"` + // Setup is the level's setup time in seconds, when it has a setup. + Setup float64 `json:"setup,omitempty"` + // Bound and SetupBound are the bounds the level was judged by, in seconds. + Bound float64 `json:"bound"` + SetupBound float64 `json:"setup_bound,omitempty"` + Slowest []TestTime `json:"slowest,omitempty"` + Failed []Failure `json:"failed,omitempty"` + // Store and Bus are what the integration level ran against: the postgres major version and the nats-server + // version (ADR 0470 §4). + // OverBound says a bound was kept no longer, and OverWhich which: setup, run or both. The level's verdict, Passed, is + // the tests' own: the checker judges the bounds (ADR 0453 §9), with the slowest tests this result names. + OverBound bool `json:"over_bound"` + OverWhich string `json:"over_which,omitempty"` + Store string `json:"store,omitempty"` + Bus string `json:"bus,omitempty"` +} + +// event is one line of `go test -json` (test2json, and the build events go test adds). +type event struct { + Action string + Package string + Test string + Output string + ImportPath string + FailedBuild string + Elapsed float64 +} + +// outputLines is how many lines of output a failure keeps. +const outputLines = 15 + +// reader collects the events of one go.mod directory's `go test -json` into a result. +type reader struct { + res *TestResult + dir string + pkgs map[string]*PackageResult + order []string + builds map[string][]string // build output by import path + pkgOut map[string][]string + testOut map[string][]string // by package + "\x00" + test + times []TestTime + failedIn map[string]bool // packages with a failed test +} + +func newReader(res *TestResult, dir string) *reader { + return &reader{res: res, dir: dir, pkgs: map[string]*PackageResult{}, builds: map[string][]string{}, + pkgOut: map[string][]string{}, testOut: map[string][]string{}, failedIn: map[string]bool{}} +} + +// read takes go test's output. Lines that are not events are kept for a failure that has nothing else to say. +func (r *reader) read(out string) (noise []string) { + sc := bufio.NewScanner(strings.NewReader(out)) + sc.Buffer(make([]byte, 1<<20), 1<<26) + for sc.Scan() { + var ev event + if err := json.Unmarshal(sc.Bytes(), &ev); err != nil { + if l := strings.TrimSpace(sc.Text()); l != "" { + noise = append(noise, l) + } + continue + } + r.event(ev) + } + return noise +} + +func (r *reader) pkg(name string) *PackageResult { + p := r.pkgs[name] + if p == nil { + p = &PackageResult{Dir: r.dir, Package: name} + r.pkgs[name] = p + r.order = append(r.order, name) + } + return p +} + +func keep(lines []string, out string) []string { + if t := strings.TrimRight(out, "\r\n"); t != "" && !strings.HasPrefix(t, "=== ") { + lines = append(lines, t) + } + return lines +} + +func (r *reader) event(ev event) { + switch { + case ev.Package == "" && ev.ImportPath != "": + if ev.Action == "build-output" { + r.builds[ev.ImportPath] = keep(r.builds[ev.ImportPath], ev.Output) + } + case ev.Package == "": + case ev.Test == "": + p := r.pkg(ev.Package) + switch ev.Action { + case "output": + r.pkgOut[ev.Package] = keep(r.pkgOut[ev.Package], ev.Output) + case "pass": + p.Passed, p.Elapsed = true, ev.Elapsed + case "fail": + p.Passed, p.Elapsed = false, ev.Elapsed + if ev.FailedBuild != "" { + r.pkgOut[ev.Package] = append(r.builds[ev.FailedBuild], r.pkgOut[ev.Package]...) + } + case "skip": + p.Passed, p.NoTests = true, true + } + default: + key := ev.Package + "\x00" + ev.Test + switch ev.Action { + case "output": + r.testOut[key] = keep(r.testOut[key], ev.Output) + case "pass", "fail", "skip": + p := r.pkg(ev.Package) + p.Tests++ + r.res.Tests++ + if ev.Action == "skip" { + r.res.Skipped++ + } + r.times = append(r.times, TestTime{Package: ev.Package, Test: ev.Test, Elapsed: ev.Elapsed}) + if ev.Action == "fail" { + r.failedIn[ev.Package] = true + r.res.Failed = append(r.res.Failed, Failure{Package: ev.Package, Test: ev.Test, Output: clip(r.testOut[key])}) + } + } + } +} + +func clip(lines []string) string { + if len(lines) > outputLines { + lines = append(lines[:outputLines:outputLines], fmt.Sprintf("... (%d more lines)", len(lines)-outputLines)) + } + return strings.Join(lines, "\n") +} + +// finish moves what was read into the result: the packages in order, a failure for each failed package that has no +// failed test to name it, and the tests' times. +func (r *reader) finish() { + for _, name := range r.order { + p := r.pkgs[name] + if !p.Passed && !r.failedIn[name] { + r.res.Failed = append(r.res.Failed, Failure{Package: name, Output: clip(r.pkgOut[name])}) + } + if !p.NoTests && p.Elapsed > r.res.Elapsed { + r.res.Elapsed = p.Elapsed + } + r.res.Packages = append(r.res.Packages, *p) + } + r.res.Slowest = append(r.res.Slowest, r.times...) +} + +// judge sets Passed and Why from what was read and the bounds. A level with no failure, over no bound, passes; with no +// test at all it says the repository has none at this level, which is neither a failure nor a warning (ADR 0445 §1). +func (res *TestResult) judge(b Bounds) { + sort.SliceStable(res.Slowest, func(i, j int) bool { return res.Slowest[i].Elapsed > res.Slowest[j].Elapsed }) + if len(res.Slowest) > 5 { + res.Slowest = res.Slowest[:5] + } + res.Bound, res.SetupBound = b.Run.Seconds(), b.Setup.Seconds() + var why, over []string + if n := len(res.Failed); n > 0 { + names := []string{} + for _, f := range res.Failed { + names = append(names, strings.TrimSpace(f.Package+" "+f.Test)) + } + why = append(why, fmt.Sprintf("%d failed: %s", n, strings.Join(names, ", "))) + } + res.Passed = len(res.Failed) == 0 + if b.Setup > 0 && res.Setup > b.Setup.Seconds() { + over = append(over, "setup") + why = append(why, fmt.Sprintf("the %s level's setup took %.2f s, over its setup bound of %s", res.Level, res.Setup, seconds(b.Setup))) + } + if b.Run > 0 && res.Elapsed > b.Run.Seconds() { + over = append(over, "run") + slow := []string{} + for _, s := range res.Slowest { + slow = append(slow, fmt.Sprintf("%s %s (%.2f s)", s.Package, s.Test, s.Elapsed)) + } + why = append(why, fmt.Sprintf("the %s level took %.2f s, over its bound of %s; the slowest tests: %s", res.Level, res.Elapsed, seconds(b.Run), strings.Join(slow, ", "))) + } + res.OverBound, res.OverWhich = len(over) > 0, strings.Join(over, ",") + res.NoTests = res.Tests == 0 && len(res.Failed) == 0 + switch { + case len(why) > 0: + res.Why = strings.Join(why, "; ") + case res.NoTests: + res.Why = "the repository has no tests at the " + res.Level + " level" + } +} + +// seconds is a duration as seconds, without trailing zeros: "5 s", "0.01 s". +func seconds(d time.Duration) string { + return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.3f", d.Seconds()), "0"), ".") + " s" +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go new file mode 100644 index 00000000..2764e81a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -0,0 +1,58 @@ +// mesh-build-go: the Go build module (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470; design 47 R4c-3, issue +// 1483). A Go tools bundle the node's runtime launches as the module's own account (runs-as). It names the one Go +// toolchain version it builds with, writes the version it finds to .go of its `toolchains` state at start and +// before every step, and refuses every step on a node that has another: its health is then unhealthy, so the +// controller raises the node's condition. What it says, it says on stderr; stdout is the MCP channel. +package main + +import ( + "context" + "fmt" + "os" + "os/user" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The directories the module's manifest places outside its home (ADR 0466 §1): the workspaces, and the Go module +// and build caches. +const ( + WorkDir = "/var/lib/mesh-build-go-work" + CacheDir = "/var/lib/mesh-build-go-cache" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-go] "+format+"\n", a...) } + +func main() { + if len(os.Args) > 1 && os.Args[1] == busArg { + os.Exit(runBus(os.Args[2:])) + } + ctx := context.Background() + m := &Module{ + Node: strings.TrimSpace(os.Getenv("MESH_NODE")), + Recipe: Current(), + Probe: ProbeGo, + Toolchains: stdio.State(delivery.ToolchainsState).Put, + Emit: func(t string, b any) error { return stdio.Emit(t, b) }, + Steps: stdio.State(delivery.StepsState).Put, + Ctx: ctx, + Say: logf, + Work: WorkDir, + Cache: CacheDir, + } + if u, err := user.Current(); err == nil { + m.Home = u.HomeDir + } + m.Run = StepRunner(m.Home) + // Outside a launch by the runtime — a build, a check — there is no node and no state to write. + if m.Node != "" { + go m.Start(ctx) + } + go startKeeper(ctx) + if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go new file mode 100644 index 00000000..e1436f55 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go @@ -0,0 +1,150 @@ +package main + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The catalogue's root and this module's directory. +const ( + catalogueRoot = "../../../.." + moduleDir = "../.." +) + +func readManifest(t *testing.T) catalogue.Manifest { + t.Helper() + m, err := catalogue.ReadTreeManifest(catalogueRoot, moduleDir) + if err != nil { + t.Fatalf("mesh-sdk's validator refuses module.json: %v", err) + } + return m +} + +func resource(t *testing.T, m catalogue.Manifest, id string) map[string]any { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + return r + } + } + t.Fatalf("module.json declares no resource %q", id) + return nil +} + +// 1483/AC1: mesh-sdk's validator accepts the module (alone on a shelf too), and it runs as mesh-build-go with its +// home at mode 0700, requires user-namespaces and declares the four packages. +func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *testing.T) { + m := readManifest(t) + if problems := catalogue.CatalogueProblems(catalogue.Shelf{m.Module: m}); len(problems) > 0 { + t.Errorf("the catalogue's rules refuse it: %v", problems) + } + if m.Module != "mesh-build-go" || m.RunsAs != "mesh-build-go" { + t.Errorf("module %q runs as %q, want mesh-build-go as itself", m.Module, m.RunsAs) + } + account := resource(t, m, "account") + if account["type"] != "user" || account["name"] != "mesh-build-go" || account["shell"] != "/usr/bin/nologin" || account["home"] != "/var/lib/mesh-build-go" { + t.Errorf("account = %v", account) + } + home := resource(t, m, "state") + if home["type"] != "directory" || home["mode"] != "0700" || home["owner"] != "mesh-build-go" || home["place"] != "." { + t.Errorf("home = %v, want its directory at mode 0700 owned by the account", home) + } + if !slices.Contains(m.Capabilities, "user-namespaces") { + t.Errorf("capabilities = %v, want user-namespaces", m.Capabilities) + } + packages := []string{} + for _, r := range m.Resources { + if r["type"] == "package" { + packages = append(packages, r["package"].(string)) + } + } + slices.Sort(packages) + if want := []string{"git", "go", "openssh", "postgresql"}; !slices.Equal(packages, want) { + t.Errorf("packages = %v, want %v", packages, want) + } +} + +// The work and cache directories are outside the home (ADR 0466 §1), owned by the account. +func TestWorkAndCacheDirectoriesLieOutsideTheHome(t *testing.T) { + m := readManifest(t) + for _, id := range []string{"work", "cache"} { + r := resource(t, m, id) + path, _ := r["path"].(string) + if path == "" || path == "/var/lib/mesh-build-go" || len(path) > len("/var/lib/mesh-build-go") && path[:len("/var/lib/mesh-build-go/")] == "/var/lib/mesh-build-go/" { + t.Errorf("%s is at %q, inside or at the home", id, path) + } + if r["owner"] != "mesh-build-go" { + t.Errorf("%s is owned by %v", id, r["owner"]) + } + } +} + +// The states are the three the module keeps, no seat is claimed (ADR 0469 §6), and the health is the module's own +// tool, which the manifest also lists among its tools. +func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) { + m := readManifest(t) + states := []string{} + for _, s := range m.State { + states = append(states, s.Name) + } + slices.Sort(states) + if want := []string{"keys", "steps", "toolchains"}; !slices.Equal(states, want) { + t.Errorf("state = %v, want %v", states, want) + } + if len(m.Claims) != 0 { + t.Errorf("claims = %v, want no seat", m.Claims) + } + if !slices.Contains(m.Tools, HealthTool) { + t.Errorf("tools = %v lack %s", m.Tools, HealthTool) + } + raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) { + t.Errorf("runtime-health does not name %s", HealthTool) + } +} + +// Every tool the manifest names is served, and every tool served is named. +func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) { + m := readManifest(t) + served := []string{} + for _, tool := range (&Module{}).Tools((&Module{}).StepTools()) { + served = append(served, tool.Name) + } + slices.Sort(served) + named := slices.Clone(m.Tools) + slices.Sort(named) + if !slices.Equal(served, named) { + t.Errorf("served %v, manifest names %v", served, named) + } +} + +// 1482/AC4: gitea's manifest reads the keys state of both build modules, and the catalogue's validator accepts it with the +// build modules beside it: each owner present declares the state read (a read of a state its owner does not keep is +// refused). +func TestGiteaReadsTheBuildModulesKeysAndTheCatalogueAcceptsThemTogether(t *testing.T) { + var all []catalogue.Manifest + var gitea catalogue.Manifest + for _, name := range []string{"gitea", "mesh-build-go", "mesh-build-node"} { + m, err := catalogue.ReadTreeManifest(catalogueRoot, filepath.Join(catalogueRoot, "modules", name)) + if err != nil { + t.Fatalf("mesh-sdk's validator refuses modules/%s/module.json: %v", name, err) + } + all = append(all, m) + if name == "gitea" { + gitea = m + } + } + for _, want := range []string{"mesh-build-go.keys", "mesh-build-node.keys", "ssh-client.keys"} { + if !slices.Contains(gitea.Reads, want) { + t.Errorf("gitea reads %v, lacks %s", gitea.Reads, want) + } + } + if problems := catalogue.StateReadsNothingDeclares(all); len(problems) > 0 { + t.Errorf("a read names a state its owner does not keep: %v", problems) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go new file mode 100644 index 00000000..5c0fbc79 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "fmt" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// CheckRound is how often the module reads the installed toolchain again, beside the check before every step and +// the one every health look makes. +const CheckRound = 10 * time.Minute + +// Module is the Go build module's one running state: the node it runs on, its recipe, the toolchain guard and the +// states it writes (toolchains, steps). +type Module struct { + Node string + Recipe Recipe + // Probe reads the installed Go version (ProbeGo). + Probe func(ctx context.Context) (string, error) + // Toolchains and Steps are the Put of the states of those names. + Toolchains, Steps buildstep.Put + // Emit tells the bus an event (stdio.Emit); nil emits nothing. The guard emits toolchain.reported after each write of + // the toolchains key, so the indexer, which holds no timer, learns that the bucket exists (issue 1444). + Emit func(eventType string, body any) error + // Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not). + Ctx context.Context + Say func(format string, a ...any) + // Run runs the commands of a step (StepRunner); Work holds the workspaces and Cache the module and build caches, + // both outside the account's home (ADR 0466 §1); Home is the account's home. + Run forgekey.Runner + Home, Work, Cache string + // UnitBound is the unit level's bound; zero is UnitBound. A test lowers it to see a level over its bound. + UnitBound time.Duration + // IntegrationBounds are the integration level's bounds; zero values are IntegrationSetup and IntegrationRun. + IntegrationBounds Bounds + // Now is the clock the levels' times are read from; time.Now when nil. + Now func() time.Time + // StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own + // program, StartBus in bus.go, when nil). + StartBus func(ctx context.Context, dir string) (url string, stop func(), err error) +} + +func (m *Module) guard() buildstep.Guard { + return buildstep.Guard{Language: "go", Want: m.Recipe.Go, Recipe: m.Recipe.Version(), Probe: m.Probe, Node: m.Node, Put: m.reportingPut()} +} + +// Check reads the installed toolchain, writes .go of the toolchains state and answers nil, or the refusal +// (buildstep.Guard.Check). It runs at start, before every step, at every health look and every CheckRound. +func (m *Module) Check(ctx context.Context) error { + return m.guard().Check(ctx) +} + +// Start checks once, as the module starts, and then every CheckRound until ctx ends. +func (m *Module) Start(ctx context.Context) { + for { + if err := m.Check(ctx); err != nil && m.Say != nil { + m.Say("the toolchain: %v", err) + } + m.CleanWorkspaces(time.Now()) + select { + case <-ctx.Done(): + return + case <-time.After(CheckRound): + } + } +} + +// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy while the toolchain installed is not +// the recipe's, saying both versions, so the controller raises the node's condition. It looks again each time it is +// asked. +func (m *Module) Health(ctx context.Context) health.Answer { + err := m.Check(ctx) + switch { + case err == nil: + return health.Healthy(fmt.Sprintf("%s installed, as the recipe %s builds with", m.Recipe.Go, m.Recipe.Version())) + default: + return health.Unhealthy(err.Error()) + } +} + +// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on +// its key in the steps state. The guard runs inside the step, before run, so a node that refuses ends every step +// refused with both versions named. +func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) { + if err := m.Check(ctx); err != nil { + return nil, err + } + return run(ctx, args) + }) + if err != nil { + return nil, err + } + key, _ := delivery.StepKey(m.Node, id) + return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil + } +} + +// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error. +func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + if err := m.Check(m.Ctx); err != nil { + return nil, err + } + return run(m.Ctx, args) + } +} + +// Status is what the module reports of itself: the node, the recipe, the toolchain found and why it refuses, if it +// does. +func (m *Module) Status(ctx context.Context) map[string]any { + err := m.Check(ctx) + installed, _ := m.Probe(ctx) + out := map[string]any{ + "node": m.Node, "recipe": m.Recipe.Version(), "go": m.Recipe.Go, "installed": installed, + "postgres-major": m.Recipe.PostgresMajor, "nats-server": m.Recipe.NatsServer, "refusing": err != nil, + } + if err != nil { + out["why"] = err.Error() + } + return out +} + +// ToolchainReported is the event the guard emits after it writes the toolchains key. +const ToolchainReported = "toolchain.reported" + +// reportingPut is the toolchains Put that then emits toolchain.reported with the node and the language. A write that +// failed emits nothing; an emit that failed is said and does not refuse the step. +func (m *Module) reportingPut() buildstep.Put { + return func(key string, value any) (uint64, error) { + rev, err := m.Toolchains(key, value) + if err != nil || m.Emit == nil { + return rev, err + } + if eerr := m.Emit(ToolchainReported, map[string]string{"node": m.Node, "language": "go"}); eerr != nil && m.Say != nil { + m.Say("the event %s could not be emitted: %v", ToolchainReported, eerr) + } + return rev, nil + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module_test.go b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go new file mode 100644 index 00000000..6e1035fe --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go @@ -0,0 +1,201 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// keptStates stands in for the toolchains and the steps states. +type keptStates struct { + mu sync.Mutex + toolchains map[string]delivery.Toolchain + steps map[string][]delivery.Step + ended chan string +} + +func newStates() *keptStates { + return &keptStates{toolchains: map[string]delivery.Toolchain{}, steps: map[string][]delivery.Step{}, ended: make(chan string, 16)} +} + +func (s *keptStates) putToolchain(key string, value any) (uint64, error) { + tc, err := delivery.DecodeToolchain(value.(json.RawMessage)) + if err != nil { + return 0, err + } + s.mu.Lock() + defer s.mu.Unlock() + s.toolchains[key] = tc + return 1, nil +} + +func (s *keptStates) putStep(key string, value any) (uint64, error) { + st, err := delivery.DecodeStep(value.(json.RawMessage)) + if err != nil { + return 0, err + } + s.mu.Lock() + s.steps[key] = append(s.steps[key], st) + s.mu.Unlock() + if st.State != delivery.StepRunning { + s.ended <- key + } + return 1, nil +} + +func (s *keptStates) toolchain(key string) (delivery.Toolchain, bool) { + s.mu.Lock() + defer s.mu.Unlock() + tc, ok := s.toolchains[key] + return tc, ok +} + +func (s *keptStates) waitEnd(t *testing.T) delivery.Step { + t.Helper() + select { + case key := <-s.ended: + s.mu.Lock() + defer s.mu.Unlock() + v := s.steps[key] + return v[len(v)-1] + case <-time.After(3 * time.Second): + t.Fatal("the step never ended on its key") + return delivery.Step{} + } +} + +func moduleWith(states *keptStates, installed string) *Module { + return &Module{Node: "ace", Recipe: Recipe{Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"}, + Probe: func(context.Context) (string, error) { return installed, nil }, + Toolchains: states.putToolchain, Steps: states.putStep, Ctx: context.Background()} +} + +// 1483/AC2: a probe answering go1.27.1: the bundle's start writes ace.go with that version and the recipe's, and +// the health tool answers healthy. +func TestStartWritesTheToolchainAndHealthIsHealthy(t *testing.T) { + states := newStates() + m := moduleWith(states, "go1.27.1") + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go m.Start(ctx) + var tc delivery.Toolchain + for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + if got, ok := states.toolchain("ace.go"); ok { + tc = got + break + } + } + if tc.Version != "go1.27.1" || tc.Recipe != m.Recipe.Version() || tc.Language != "go" { + t.Fatalf("ace.go = %+v, want go1.27.1 and recipe %s", tc, m.Recipe.Version()) + } + if a := m.Health(ctx); a.State != health.StateHealthy || !a.Healthy { + t.Errorf("health = %+v, want healthy", a) + } +} + +// 1483/AC3: a probe answering go1.27.2: every step tool's step ends refused naming both versions, ace.go holds +// go1.27.2, and the health tool answers unhealthy naming both. +func TestAnotherGoRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) { + states := newStates() + m := moduleWith(states, "go1.27.2") + ran := false + run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil } + steps := append([]StepTool{ + {Name: "long_probe", Long: true, Run: run}, + {Name: "short_probe", Run: run}, + }, m.StepTools()...) + for _, tool := range m.Tools(steps) { + if tool.Name == StatusTool || tool.Name == HealthTool { + continue + } + var reason string + out, err := tool.Run(map[string]any{}) + if isLong(steps, tool.Name) { + if err != nil { + t.Fatalf("%s: %v", tool.Name, err) + } + if out.(map[string]any)["step"] == "" { + t.Fatalf("%s answered no step id: %v", tool.Name, out) + } + st := states.waitEnd(t) + if st.State != delivery.StepRefused { + t.Errorf("%s: the step ended %s, want refused", tool.Name, st.State) + } + reason = st.Reason + } else { + if err == nil { + t.Fatalf("%s: answered %v, want a refusal", tool.Name, out) + } + reason = err.Error() + } + if !strings.Contains(reason, "go1.27.2") || !strings.Contains(reason, "go1.27.1") { + t.Errorf("%s: the refusal %q does not name both versions", tool.Name, reason) + } + } + if ran { + t.Errorf("a step ran on a node that refuses") + } + if tc, _ := states.toolchain("ace.go"); tc.Version != "go1.27.2" { + t.Errorf("ace.go holds %q, want the installed go1.27.2", tc.Version) + } + a := m.Health(context.Background()) + if a.State != health.StateUnhealthy || !strings.Contains(a.Why, "go1.27.2") || !strings.Contains(a.Why, "go1.27.1") { + t.Errorf("health = %+v, want unhealthy naming both versions", a) + } +} + +func isLong(steps []StepTool, name string) bool { + for _, s := range steps { + if s.Name == name { + return s.Long + } + } + return false +} + +// 1483/AC4: recipes differing only in the postgres major or the nats-server version have different recipe versions; +// equal recipes have the same one. +func TestRecipeVersionMovesWithEachOfItsValues(t *testing.T) { + base := Recipe{Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"} + if base.Version() != (Recipe{Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"}).Version() { + t.Errorf("one recipe has two versions") + } + seen := map[string]string{base.Version(): "base"} + for name, r := range map[string]Recipe{ + "postgres major": {Steps: "1", Go: "go1.27.1", PostgresMajor: 19, NatsServer: "2.11.17"}, + "nats-server": {Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.18"}, + "go": {Steps: "1", Go: "go1.27.2", PostgresMajor: 18, NatsServer: "2.11.17"}, + "steps": {Steps: "2", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"}, + } { + v := r.Version() + if was, dup := seen[v]; dup { + t.Errorf("the recipe differing in %s has the version of the %s recipe: %s", name, was, v) + } + seen[v] = name + } +} + +func TestTheRecipeNamesGo1271AndPostgres18AndTheLibrarysNatsVersion(t *testing.T) { + r := Current() + if r.Go != "go1.27.1" || r.PostgresMajor != 18 { + t.Errorf("recipe = %+v", r) + } + if r.NatsServer == "" || strings.HasPrefix(r.NatsServer, "v") { + t.Errorf("nats-server version = %q", r.NatsServer) + } +} + +func TestParseGoVersion(t *testing.T) { + if v, err := parseGoVersion("go version go1.27.1 linux/amd64\n"); err != nil || v != "go1.27.1" { + t.Errorf("= %q, %v", v, err) + } + if _, err := parseGoVersion("not go"); err == nil { + t.Errorf("accepted a text that is not `go version`'s") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/probe.go b/modules/mesh-build-go/cmd/mesh-build-go/probe.go new file mode 100644 index 00000000..838ece4d --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/probe.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +// ProbeTimeout bounds `go version`. +const ProbeTimeout = 15 * time.Second + +// ProbeGo reads the installed Go version: the third word `go version` prints ("go version go1.27.1 linux/amd64"), +// as the recipe's RecipeGo is written. GOTOOLCHAIN=local keeps an older Go from fetching another toolchain: what is +// installed is what is read. +func ProbeGo(ctx context.Context) (string, error) { + ctx, cancel := context.WithTimeout(ctx, ProbeTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "go", "version") + cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local") + out, err := cmd.Output() + if err != nil { + return "", err + } + return parseGoVersion(string(out)) +} + +func parseGoVersion(out string) (string, error) { + f := strings.Fields(out) + if len(f) < 3 || f[0] != "go" || f[1] != "version" || !strings.HasPrefix(f[2], "go") { + return "", fmt.Errorf("`go version` answered %q", strings.TrimSpace(out)) + } + return f[2], nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/recipe.go b/modules/mesh-build-go/cmd/mesh-build-go/recipe.go new file mode 100644 index 00000000..0303045e --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/recipe.go @@ -0,0 +1,87 @@ +package main + +// The recipe of the Go build module (novox/hq ADR 0405 §1 and §7, ADR 0469 §1, ADR 0470 §4; issue 1483): the one +// Go toolchain version it builds with, the one postgres major version its integration level tests with, and the +// recipe's own version. A node whose installed Go differs from the recipe's refuses every step (guard.go), so all +// builders run this version or none builds. + +import ( + "crypto/sha256" + "encoding/hex" + "runtime/debug" + "strconv" + "strings" + + "github.com/nats-io/nats-server/v2/server" +) + +// RecipeGo is the Go version the recipe builds with: the text `go version` prints for it, which is what the guard +// compares and what the indexer reads from .go of the toolchains state. +const RecipeGo = "go1.27.1" + +// PostgresMajor is the postgres major version the integration level tests with (ADR 0470 §4). +const PostgresMajor = 18 + +// RecipeSteps is the recipe's own version: changed whenever the steps of fetch, test or build change in a way +// that can change what they produce (ADR 0405 §7). It is one of the four values the recipe version hashes. +const RecipeSteps = "1" + +// natsModule is the library the integration level's bus is started from (ADR 0470 §2). +const natsModule = "github.com/nats-io/nats-server/v2" + +// Recipe is every value that makes up the recipe version. +type Recipe struct { + // Steps is RecipeSteps. + Steps string + // Go is the toolchain's version as `go version` prints it. + Go string + // PostgresMajor is the integration level's postgres major version. + PostgresMajor int + // NatsServer is the nats-server library's version, from the program's build information. + NatsServer string +} + +// Current is the recipe of this build of the module. +func Current() Recipe { + return Recipe{Steps: RecipeSteps, Go: RecipeGo, PostgresMajor: PostgresMajor, NatsServer: natsServerVersion()} +} + +// Version is the recipe version: a hash of the recipe's own version, the Go version, the postgres major version and +// the nats-server version, so a change of any of the four moves it and with it every fingerprint that holds it +// (ADR 0405 §7, ADR 0465 §2). The fields are named in the hashed text, so no two recipes share a text. +func (r Recipe) Version() string { + h := sha256.New() + for _, f := range [][2]string{ + {"steps", r.Steps}, {"go", r.Go}, {"postgres", strconv.Itoa(r.PostgresMajor)}, {"nats-server", r.NatsServer}, + } { + h.Write([]byte(f[0] + "=" + f[1] + "\n")) + } + return hex.EncodeToString(h.Sum(nil))[:16] +} + +// natsServerVersion is the version of the nats-server library this program was built with: its build information +// names it as the module's go.mod does, and has no second source (ADR 0470 §4). A binary built without build +// information (a stripped test binary) falls back to the library's own version constant, which is the same text. +func natsServerVersion() string { + if info, ok := debug.ReadBuildInfo(); ok { + if v := natsVersionOf(info); v != "" { + return v + } + } + return server.VERSION +} + +// natsVersionOf is the version build information gives the nats-server library, "v" stripped, or "" when it lists +// none. A replaced library is the replacement's version. +func natsVersionOf(info *debug.BuildInfo) string { + for _, d := range info.Deps { + if d.Path != natsModule { + continue + } + if d.Replace != nil { + d = d.Replace + } + return strings.TrimPrefix(d.Version, "v") + } + return "" +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/runner.go b/modules/mesh-build-go/cmd/mesh-build-go/runner.go new file mode 100644 index 00000000..691ef6f5 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/runner.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "os" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// StepRunner runs the commands of a build step with an environment built from scratch (novox/hq ADR 0466 §4, issue +// 1485): PATH, HOME (the account's own home, where its ssh files and git configuration are) and a fixed locale, plus +// the words the caller passes, and nothing else of this process's environment — which holds the module's bus +// credentials and might hold a password or a token. It is mesh-sdk's clean runner (ExecRunnerClean, issue 1497) with +// that base given; a command is bound by its context alone (the bound 0), so a fetch may outlast a tool call's thirty +// seconds (the step has its own context). +func StepRunner(home string) forgekey.Runner { + run := forgekey.ExecRunnerClean(0) + return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + return run(ctx, append(baseEnv(home), env...), name, args...) + } +} + +// defaultPath is where a step looks for git, ssh and go when this process was given no PATH. +const defaultPath = "/usr/local/bin:/usr/bin:/bin" + +func baseEnv(home string) []string { + path := os.Getenv("PATH") + if path == "" { + path = defaultPath + } + return []string{"PATH=" + path, "HOME=" + home, "LC_ALL=C"} +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/store.go b/modules/mesh-build-go/cmd/mesh-build-go/store.go new file mode 100644 index 00000000..db64d76e --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/store.go @@ -0,0 +1,100 @@ +package main + +// The integration level's throwaway store (novox/hq ADR 0470 §1; issue 1486): one postgres server of the installed +// package for each run, `initdb` into the run's own directory, listening on a unix socket in that directory and on no +// TCP address, as the module's account, with durability off as today's check sets it. + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "strconv" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// storeStopBound is how long stopping the store is given: it runs after the level, whatever the level did. +const storeStopBound = 30 * time.Second + +var postgresVersion = regexp.MustCompile(`PostgreSQL\)\s+(\d+)`) + +// postgresMajor is the major version of the installed postgres: `postgres --version` ("postgres (PostgreSQL) 18.6"). +func (m *Module) postgresMajor(ctx context.Context) (int, error) { + r := m.Run(ctx, nil, "postgres", "--version") + if r.Status != 0 || r.Err != "" { + return 0, fmt.Errorf("`postgres --version`: %s", firstLine(r.Err+" "+r.Stderr)) + } + f := postgresVersion.FindStringSubmatch(r.Stdout) + if f == nil { + return 0, fmt.Errorf("`postgres --version` answered %q", firstLine(r.Stdout)) + } + return strconv.Atoi(f[1]) +} + +// checkPostgres refuses the integration level when the installed postgres major is not the recipe's, naming both +// (ADR 0470 §4); the unit level is not refused. It answers the major it found. +func (m *Module) checkPostgres(ctx context.Context) (int, error) { + major, err := m.postgresMajor(ctx) + if err != nil { + return 0, &buildstep.Refusal{Text: fmt.Sprintf("the installed postgres version could not be read: %v", err), Want: strconv.Itoa(m.Recipe.PostgresMajor)} + } + if major != m.Recipe.PostgresMajor { + return major, &buildstep.Refusal{Text: fmt.Sprintf("postgres %d installed, the integration level is built for postgres %d", major, m.Recipe.PostgresMajor), + Want: strconv.Itoa(m.Recipe.PostgresMajor), Have: strconv.Itoa(major)} + } + return major, nil +} + +// socketLimit is the longest path of a socket postgres accepts, with the file's name: it refuses 108 bytes and over. +const socketLimit = 100 + +// socketName is the file postgres makes in the socket directory. +const socketName = "/.s.PGSQL.5432" + +// socketDir makes the directory of the store's socket: under the run when the socket's path stays under socketLimit, +// else a short directory of its own under /tmp (answering short, so the caller removes it). A unix socket's path is +// bounded far below a file's, and a work directory deep in a data root is over it. +func socketDir(run string) (dir string, short bool, err error) { + dir = filepath.Join(run, "pg", "socket") + if len(dir)+len(socketName) < socketLimit { + return dir, false, os.MkdirAll(dir, 0o700) + } + dir, err = os.MkdirTemp("/tmp", "pg") + return dir, true, err +} + +// storeSettings are the server's own settings: no TCP address, a socket directory, and no durability. +func storeOptions(socket string) string { + return "-k " + socket + " -c listen_addresses='' -c fsync=off -c synchronous_commit=off -c full_page_writes=off" +} + +// startStore makes and starts the run's postgres in run/pg and answers the address tests are given as +// MESH_TEST_POSTGRES, and what stops it. The address names the socket directory percent-encoded, as the controller's +// tests cut the URL at its last slash to name their own databases. +func (m *Module) startStore(ctx context.Context, run string) (string, func(), error) { + data := filepath.Join(run, "pg", "data") + socket, short, err := socketDir(run) + if err != nil { + return "", nil, err + } + stop := func() { + if short { + defer os.RemoveAll(socket) + } + sctx, cancel := context.WithTimeout(context.Background(), storeStopBound) + defer cancel() + m.Run(sctx, nil, "pg_ctl", "-D", data, "-m", "immediate", "-w", "stop") + } + if r := m.Run(ctx, nil, "initdb", "-D", data, "-U", "postgres", "--auth=trust", "-E", "UTF8", "--no-locale"); r.Status != 0 || r.Err != "" { + return "", nil, fmt.Errorf("initdb: %s", firstLine(r.Err+" "+r.Stderr+" "+r.Stdout)) + } + if r := m.Run(ctx, nil, "pg_ctl", "-D", data, "-l", filepath.Join(run, "pg", "log"), "-w", "-t", "30", "-o", storeOptions(socket), "start"); r.Status != 0 || r.Err != "" { + stop() // a server that was half started is stopped + return "", nil, fmt.Errorf("pg_ctl start: %s", firstLine(r.Err+" "+r.Stderr+" "+r.Stdout)) + } + return "postgres://postgres@/postgres?host=" + url.QueryEscape(socket) + "&sslmode=disable", stop, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go new file mode 100644 index 00000000..426cdd44 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -0,0 +1,143 @@ +package main + +// The test step (novox/hq ADR 0445; issues 1290 and 1486): a fetched workspace's tests by Go's convention, one level +// at a time, each judged by its own bounds. The unit level is `go test` over every package without the `integration` +// build tag. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// testTimeout is go test's own timeout for a level: a bound on a test that never ends, far above any level's bound. +const testTimeout = "10m" + +var packageOK = regexp.MustCompile(`^(\./|[A-Za-z0-9])[A-Za-z0-9._/@~-]*(/\.\.\.)?$`) + +func (m *Module) testTool() StepTool { + return StepTool{ + Name: "build_go_test", + Description: "Run the tests of a fetched workspace by Go's convention, one level: the unit level is go test over every " + + "package without the integration build tag, within five seconds. Answers a step id at once; the step ends on " + + "its key with the level's result: its packages, its times, the tests that failed and the slowest tests. A " + + "level that fails is a finished step whose result says passed false.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, + "level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"}, + "packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, + "description": "package patterns of each module, ./... when none"}, + }, + "required": []string{"workspace", "level"}, + }, + Long: true, + Run: m.test, + } +} + +// unitBound is the unit level's bound: UnitBound unless a test lowered it. +func (m *Module) unitBound() time.Duration { + if m.UnitBound > 0 { + return m.UnitBound + } + return UnitBound +} + +func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { + workspace, _ := args["workspace"].(string) + level, _ := args["level"].(string) + packages, err := packagesArg(args["packages"]) + if err != nil { + return nil, err + } + if !workspaceOK.MatchString(workspace) { + return nil, fmt.Errorf("%q is not a workspace id", workspace) + } + tree := filepath.Join(m.Work, workspace, "tree") + if _, err := os.Stat(tree); err != nil { + return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace) + } + switch level { + case LevelUnit: + return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}}) + case LevelIntegration: + return m.integration(ctx, workspace, tree, packages) + } + return nil, fmt.Errorf("the level %q is neither unit nor integration", level) +} + +func packagesArg(v any) ([]string, error) { + if v == nil { + return nil, nil + } + list, ok := v.([]any) + if !ok { + return nil, fmt.Errorf("packages is a list of package patterns") + } + out := []string{} + for _, e := range list { + s, _ := e.(string) + if !packageOK.MatchString(s) || strings.Contains(strings.TrimSuffix(s, "/..."), "..") { + return nil, fmt.Errorf("%q is not a package pattern", s) + } + out = append(out, s) + } + return out, nil +} + +// levelOpts are what a level asks of runLevel. +type levelOpts struct { + level string + tags []string + packages []string + bounds Bounds + // env is what the level adds to a go command's environment (MESH_TEST_POSTGRES, MESH_TEST_NATS). + env []string + // setup is the time the level's setup took, which the caller measured; wall says the level's running time is the + // wall time of its go test commands, as the integration level's packages run side by side by go test's own + // parallelism, where the unit level's is the slowest package's. + setup time.Duration + wall bool + // compose is a state of the level to put in the result (the store and the bus it ran against). + store, bus string +} + +// runLevel runs `go test -json` in each go.mod directory of the tree and judges the whole against the bounds. +func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestResult, error) { + dirs, err := m.goModDirs(ctx, tree) + if err != nil { + return nil, err + } + res := &TestResult{Level: o.level, Packages: []PackageResult{}, Setup: o.setup.Seconds(), Store: o.store, Bus: o.bus} + packages := o.packages + if len(packages) == 0 { + packages = []string{"./..."} + } + began := m.clock() + for _, d := range dirs { + args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout} + if len(o.tags) > 0 { + args = append(args, "-tags", strings.Join(o.tags, ",")) + } + args = append(args, packages...) + r := m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...) + rd := newReader(res, d) + noise := rd.read(r.Stdout) + rd.finish() + if (r.Status != 0 || r.Err != "") && len(rd.pkgs) == 0 { + // go test said nothing a test said: it could not run (a go.mod it cannot read, a toolchain it cannot find). + return nil, fmt.Errorf("go test in %s: %s", d, firstLine(r.Err+"\n"+strings.Join(noise, "\n")+"\n"+r.Stderr)) + } + } + if o.wall { + res.Elapsed = m.clock().Sub(began).Seconds() + } + res.judge(o.bounds) + return res, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go new file mode 100644 index 00000000..f6bdf6a4 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go @@ -0,0 +1,178 @@ +//go:build integration + +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// TestMain makes this test binary the bus when the module starts it as one (bus.go). +func TestMain(m *testing.M) { + if len(os.Args) > 1 && os.Args[1] == busArg { + os.Exit(runBus(os.Args[2:])) + } + os.Exit(m.Run()) +} + +// needTools skips, loudly, a test whose tools are not installed where the check runs (the mesh's Go toolchain image +// holds no postgres): the level is proven live on a build node, which has them (issue 1486). +func needTools(t *testing.T, what string, tools ...string) { + t.Helper() + for _, tool := range tools { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("NOT RUN: no %s installed where this check runs (%s); proven live on a build node", tool, what) + } + } +} + +// repository copies a fixture repository into a git tree, as a fetch leaves one. +func repository(t *testing.T, name string) string { + t.Helper() + tree := t.TempDir() + if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", name)+"/.", tree).CombinedOutput(); err != nil { + t.Fatalf("%s: %v %s", name, err, out) + } + for _, c := range [][]string{{"init", "-q"}, {"add", "-A"}, {"-c", "user.email=a@b", "-c", "user.name=x", "commit", "-qm", "x"}} { + if out, err := exec.Command("git", append([]string{"-C", tree}, c...)...).CombinedOutput(); err != nil { + t.Fatalf("%s: git %v: %v %s", name, c, err, out) + } + } + return tree +} + +// Issue 1290: the unit level against the real go on the fixture repositories in testdata/repos: the tests of +// test_test.go read the output captured from them, and this one proves the capture is what the real go says today. It +// starts go test, so it carries the tag. +func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) { + needTools(t, "the unit level on fixtures", "go", "git") + want := map[string]struct { + passed, noTests bool + failed string + }{ + "failing": {failed: "fixture/failing/bad"}, + "tagged": {passed: true, noTests: true}, + "empty": {passed: true, noTests: true}, + "slow": {passed: true}, + "broken": {failed: "fixture/broken"}, + } + for name, w := range want { + tree := repository(t, name) + home := t.TempDir() + m := &Module{Home: home, Work: t.TempDir(), Cache: t.TempDir()} + m.Run = StepRunner(home) + res, err := m.runLevel(context.Background(), tree, levelOpts{level: LevelUnit, bounds: Bounds{Run: UnitBound}}) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if res.Passed != w.passed || res.NoTests != w.noTests || (w.failed != "" && (len(res.Failed) != 1 || res.Failed[0].Package != w.failed)) { + t.Errorf("%s: result = %+v", name, res) + } + } +} + +// processesNamed are the processes whose command line holds text. +func processesNamed(text string) []string { + var out []string + cmdlines, _ := filepath.Glob("/proc/[0-9]*/cmdline") + for _, f := range cmdlines { + if raw, err := os.ReadFile(f); err == nil && strings.Contains(string(raw), text) { + out = append(out, f) + } + } + return out +} + +// 1486/AC1: given a fixture repository whose integration test opens MESH_TEST_POSTGRES and MESH_TEST_NATS, the +// integration level passes, and afterwards no postgres or bus process of the run is left and its directory is gone. +// It starts postgres and a bus, so it runs in the merge check and in a try, never on the way. +func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.T) { + needTools(t, "the integration level", "go", "git", "initdb", "pg_ctl", "postgres") + tree := repository(t, "store") + // A short work directory: a unix socket's path is bounded at 108 bytes. + work, err := os.MkdirTemp("/tmp", "w") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(work) }) + if err := os.MkdirAll(filepath.Join(work, "ws1"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Rename(tree, filepath.Join(work, "ws1", "tree")); err != nil { + t.Fatal(err) + } + home := t.TempDir() + m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()} + m.Run = StepRunner(home) + res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil) + if err != nil { + t.Fatalf("the integration level: %v", err) + } + if !res.Passed || res.Tests != 1 || res.Setup <= 0 || res.Setup > IntegrationSetup.Seconds() { + t.Errorf("result = %+v", res) + } + entries, _ := os.ReadDir(filepath.Join(work, "ws1")) + if len(entries) != 1 || entries[0].Name() != "tree" { + t.Errorf("the workspace holds %v, want only its tree: the run directory is gone", entries) + } + for deadline := time.Now().Add(5 * time.Second); len(processesNamed(work)) > 0 && time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + } + if left := processesNamed(work); len(left) > 0 { + t.Errorf("processes of the run are left: %v", left) + } +} + +// 1487/AC2: given a fixture Go module, when it is built twice, both digests are equal and the archive holds one +// executable named as binaryName says. It runs the real go, so it carries the tag. +func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) { + needTools(t, "the build", "go") + work := t.TempDir() + tree := filepath.Join(work, "ws1", "tree") + if err := os.MkdirAll(tree, 0o755); err != nil { + t.Fatal(err) + } + if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", "build")+"/.", tree).CombinedOutput(); err != nil { + t.Fatalf("%v %s", err, out) + } + home := t.TempDir() + m := &Module{Home: home, Work: work, Cache: t.TempDir()} + m.Run = StepRunner(home) + args := map[string]any{"workspace": "ws1", "module": ".", "artifact": "tools"} + first, err := m.build(context.Background(), args) + if err != nil { + t.Fatal(err) + } + second, err := m.build(context.Background(), args) + if err != nil { + t.Fatal(err) + } + a, b := first.(BuildResult), second.(BuildResult) + if a.Digest != b.Digest || a.Binary != "prog-bin" { + t.Errorf("two builds of one tree: %+v and %+v", a, b) + } + raw, err := os.ReadFile(a.Path) + if err != nil { + t.Fatal(err) + } + zr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + tr := tar.NewReader(zr) + h, err := tr.Next() + if err != nil || h.Name != "prog-bin" || h.Mode&0o111 == 0 { + t.Fatalf("first entry = %+v, %v", h, err) + } + if _, err := tr.Next(); err != io.EOF { + t.Errorf("the archive holds more than one file") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go new file mode 100644 index 00000000..c13e8085 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go @@ -0,0 +1,243 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// capturedRunner answers the commands of a test step from a fixture repository: ls-files with the go.mod, and every +// `go test` with the output captured from the real go (testdata/captured, see its README). +func capturedRunner(t *testing.T, captured string, calls *[]cmdRecord) forgekey.Runner { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "captured", captured+".json")) + if err != nil { + t.Fatal(err) + } + status := 0 + if strings.Contains(string(raw), `"Action":"fail"`) { + status = 1 // go test exits 1 when a test or a build failed + } + return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + *calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")}) + switch { + case name == "git" && args[2] == "ls-files": + return forgekey.Ran{Stdout: "go.mod\n"} + case name == "go": + return forgekey.Ran{Stdout: string(raw), Status: status} + } + return forgekey.Ran{} + } +} + +// workspaceFor is a module whose work directory holds a fetched workspace, and the id of it. +func workspaceFor(t *testing.T, captured string) (*Module, *keptStates, string, *[]cmdRecord) { + t.Helper() + calls := &[]cmdRecord{} + states := newStates() + m := moduleWith(states, "go1.27.1") + m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir() + m.Run = capturedRunner(t, captured, calls) + if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil { + t.Fatal(err) + } + return m, states, "ws1", calls +} + +func runUnit(t *testing.T, m *Module, states *keptStates, ws string) (TestResult, delivery.Step) { + t.Helper() + var run func(map[string]any) (any, error) + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_test" { + run = tool.Run + } + } + if _, err := run(map[string]any{"workspace": ws, "level": "unit"}); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + var res TestResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatalf("result %s: %v", st.Result, err) + } + } + return res, st +} + +// 1290/AC1: a fixture with one passing package and one failing test: the result names the failing test and its +// package, and the level fails. The step itself finished, so it ends passed with the verdict in its result. +func TestUnitLevelNamesTheFailingTestAndFailsTheLevel(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "failing") + res, st := runUnit(t, m, states, ws) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if res.Passed || res.Level != "unit" { + t.Errorf("result = %+v, want the unit level failed", res) + } + if len(res.Failed) != 1 || res.Failed[0].Package != "fixture/failing/bad" || res.Failed[0].Test != "TestThreeIsFour" || + !strings.Contains(res.Failed[0].Output, "Three() = 3, want 4") { + t.Errorf("failed = %+v", res.Failed) + } + if !strings.Contains(res.Why, "fixture/failing/bad TestThreeIsFour") { + t.Errorf("why = %q", res.Why) + } + good := false + for _, p := range res.Packages { + if p.Package == "fixture/failing/good" && p.Passed && p.Tests == 1 { + good = true + } + } + if !good || res.Tests != 2 { + t.Errorf("packages = %+v, tests = %d; want the good package passed with its test, 2 tests", res.Packages, res.Tests) + } +} + +// 1290/AC2: a fixture whose tests take more than the bound (lowered by the module's option, not by waiting five +// seconds): the level fails naming the bound and the slowest tests; under the real bound the same run passes. +func TestUnitLevelOverItsBoundFailsNamingTheBoundAndTheSlowestTests(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "slow") + m.UnitBound = 10 * time.Millisecond + res, _ := runUnit(t, m, states, ws) + if !res.Passed || !res.OverBound || res.OverWhich != "run" { + t.Fatalf("a level of %.3f s under a bound of %v: passed %v, over_bound %v (%s); the tests passed and the bound was kept no longer", res.Elapsed, m.UnitBound, res.Passed, res.OverBound, res.OverWhich) + } + for _, want := range []string{"over its bound of 0.01 s", "fixture/slow TestTakesAWhile"} { + if !strings.Contains(res.Why, want) { + t.Errorf("why = %q lacks %q", res.Why, want) + } + } + if len(res.Slowest) == 0 || res.Slowest[0].Test != "TestTakesAWhile" || res.Bound != 0.01 { + t.Errorf("slowest = %+v, bound = %v", res.Slowest, res.Bound) + } + + m2, states2, ws2, _ := workspaceFor(t, "slow") + if res, _ := runUnit(t, m2, states2, ws2); !res.Passed || res.OverBound || res.Bound != 5 { + t.Errorf("under the bound of five seconds: %+v", res) + } +} + +// 1290/AC3: a fixture whose only test carries //go:build integration: go test says [no test files], and the unit +// level reports no tests and passes. No command of the unit level asks for the tag. +func TestUnitLevelOfAFixtureWithOnlyAnIntegrationTestHasNoTests(t *testing.T) { + m, states, ws, calls := workspaceFor(t, "tagged") + res, _ := runUnit(t, m, states, ws) + if !res.Passed || !res.NoTests || res.Tests != 0 || !strings.Contains(res.Why, "no tests") { + t.Errorf("result = %+v, want no tests, passed", res) + } + for _, c := range *calls { + if strings.Contains(c.line, "-tags") { + t.Errorf("the unit level asked for a tag: %s", c.line) + } + } +} + +// 1290/AC4: a fixture with no tests at all: the result says the repository has no tests, neither a failure nor a warning. +func TestUnitLevelOfAFixtureWithNoTestsSaysSo(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "empty") + res, st := runUnit(t, m, states, ws) + if st.State != delivery.StepPassed || !res.Passed || !res.NoTests || res.Why != "the repository has no tests at the unit level" || len(res.Failed) != 0 { + t.Errorf("result = %+v (%s)", res, st.State) + } +} + +// A package that does not build fails the level with the compiler's words. +func TestUnitLevelOfAPackageThatDoesNotBuildFailsWithTheCompilersLine(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "broken") + res, _ := runUnit(t, m, states, ws) + if res.Passed || len(res.Failed) != 1 || res.Failed[0].Package != "fixture/broken" || + !strings.Contains(res.Failed[0].Output, `cannot use "one" (untyped string constant) as int value`) { + t.Errorf("result = %+v", res) + } +} + +// The command is go test -json in the module's directory of the workspace, with the toolchain pinned to the installed +// one and the module's caches. +func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *testing.T) { + m, states, ws, calls := workspaceFor(t, "empty") + runUnit(t, m, states, ws) + var line string + var env string + for _, c := range *calls { + if strings.HasPrefix(c.line, "go ") { + line, env = c.line, strings.Join(c.env, " ") + } + } + want := "go -C " + filepath.Join(m.Work, ws, "tree", ".") + " test -json -count=1 -timeout 10m ./..." + if line != want { + t.Errorf("command = %q, want %q", line, want) + } + for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod")} { + if !strings.Contains(env, w) { + t.Errorf("environment %s lacks %s", env, w) + } + } +} + +// A go test that said nothing a test says (it could not run) ends the step failed, not the level. +func TestAGoTestThatCouldNotRunEndsTheStepFailed(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "empty") + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + if name == "go" { + return forgekey.Ran{Status: 1, Stderr: "go: cannot find main module\n"} + } + return forgekey.Ran{Stdout: "go.mod\n"} + } + _, st := runUnit(t, m, states, ws) + if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "go: cannot find main module") { + t.Errorf("the step ended %s: %q", st.State, st.Reason) + } +} + +func TestATestOfAnUnknownWorkspaceOrAnotherLevelIsRefused(t *testing.T) { + m, states, _, _ := workspaceFor(t, "empty") + var run func(map[string]any) (any, error) + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_test" { + run = tool.Run + } + } + for _, args := range []map[string]any{ + {"workspace": "nope", "level": "unit"}, + {"workspace": "../ws1", "level": "unit"}, + {"workspace": "ws1", "level": "stress"}, + {"workspace": "ws1", "level": "unit", "packages": []any{"-run=x"}}, + {"workspace": "ws1", "level": "unit", "packages": []any{"../x"}}, + } { + run(args) + if st := states.waitEnd(t); st.State != delivery.StepFailed { + t.Errorf("%v: the step ended %s", args, st.State) + } + } +} + +// A directory that vendors is tested, and listed, from its vendor directory: fetch downloaded nothing for it, and the +// last value of GOFLAGS is the one go uses. +func TestADirectoryThatVendorsIsReadFromItsVendorDirectory(t *testing.T) { + m, states, ws, calls := workspaceFor(t, "empty") + vendor := filepath.Join(m.Work, ws, "tree", "vendor") + os.MkdirAll(vendor, 0o755) + os.WriteFile(filepath.Join(vendor, "modules.txt"), nil, 0o644) + runUnit(t, m, states, ws) + flags := "" + for _, c := range *calls { + if strings.HasPrefix(c.line, "go ") { + for _, w := range c.env { + if strings.HasPrefix(w, "GOFLAGS=") { + flags = w + } + } + } + } + if flags != "GOFLAGS=-mod=vendor" { + t.Errorf("the last GOFLAGS of go test in a vendoring directory is %q", flags) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README new file mode 100644 index 00000000..5d207ada --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README @@ -0,0 +1,10 @@ +Each file is the standard output of `GOTOOLCHAIN=local GOFLAGS=-mod=mod go test -count=1 -json ./...` (go1.27.1) +run in the fixture module of the same name under ../repos, captured on 2026-10-20 for issue 1290. The unit +tests feed them to the parser through a fake runner, so no test here runs `go test`. + +golist-external.json is `GOTOOLCHAIN=local GOFLAGS=-mod=mod go list -json=ImportPath,Standard,DepOnly,Module +git.novox.be/novox/mesh-sdk/go/buildstep` run in this module (go1.27.1, 2026-10-20 for issue 1291): a package of a module +outside the tree as go list describes it. + +toolchain-go-entry.txt is lines 155-191 of mesh-control's internal/builder/toolchain.go at origin/main 073ccc29 (fetched +2026-10-20): the Go entry of today's build agent's recipe, which the build step's flags are compared with. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/broken.json b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/broken.json new file mode 100644 index 00000000..10a616fd --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/broken.json @@ -0,0 +1,6 @@ +{"ImportPath":"fixture/broken [fixture/broken.test]","Action":"build-output","Output":"# fixture/broken [fixture/broken.test]\n"} +{"ImportPath":"fixture/broken [fixture/broken.test]","Action":"build-output","Output":"./broken.go:3:25: cannot use \"one\" (untyped string constant) as int value in return statement\n"} +{"ImportPath":"fixture/broken [fixture/broken.test]","Action":"build-fail"} +{"Time":"2026-10-20T22:20:06.344233803+02:00","Action":"start","Package":"fixture/broken"} +{"Time":"2026-10-20T22:20:06.344344815+02:00","Action":"output","Package":"fixture/broken","Output":"FAIL\tfixture/broken [build failed]\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.344358865+02:00","Action":"fail","Package":"fixture/broken","Elapsed":0,"FailedBuild":"fixture/broken [fixture/broken.test]"} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/empty.json b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/empty.json new file mode 100644 index 00000000..2e91a490 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/empty.json @@ -0,0 +1,3 @@ +{"Time":"2026-10-20T22:20:06.064193534+02:00","Action":"start","Package":"fixture/empty"} +{"Time":"2026-10-20T22:20:06.064489419+02:00","Action":"output","Package":"fixture/empty","Output":"? \tfixture/empty\t[no test files]\n"} +{"Time":"2026-10-20T22:20:06.064510249+02:00","Action":"skip","Package":"fixture/empty","Elapsed":0} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/failing.json b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/failing.json new file mode 100644 index 00000000..edfe50fc --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/failing.json @@ -0,0 +1,17 @@ +{"Time":"2026-10-20T22:20:06.028267402+02:00","Action":"start","Package":"fixture/failing/bad"} +{"Time":"2026-10-20T22:20:06.028528997+02:00","Action":"start","Package":"fixture/failing/good"} +{"Time":"2026-10-20T22:20:06.030412986+02:00","Action":"run","Package":"fixture/failing/bad","Test":"TestThreeIsFour"} +{"Time":"2026-10-20T22:20:06.030453876+02:00","Action":"output","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Output":"=== RUN TestThreeIsFour\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.030494057+02:00","Action":"output","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Output":" bad_test.go:7: Three() = 3, want 4\n","OutputType":"error"} +{"Time":"2026-10-20T22:20:06.030511077+02:00","Action":"output","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Output":"--- FAIL: TestThreeIsFour (0.00s)\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.030519017+02:00","Action":"fail","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Elapsed":0} +{"Time":"2026-10-20T22:20:06.030529158+02:00","Action":"output","Package":"fixture/failing/bad","Output":"FAIL\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.030546038+02:00","Action":"run","Package":"fixture/failing/good","Test":"TestTwo"} +{"Time":"2026-10-20T22:20:06.030561528+02:00","Action":"output","Package":"fixture/failing/good","Test":"TestTwo","Output":"=== RUN TestTwo\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.030570028+02:00","Action":"output","Package":"fixture/failing/good","Test":"TestTwo","Output":"--- PASS: TestTwo (0.00s)\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.030577128+02:00","Action":"pass","Package":"fixture/failing/good","Test":"TestTwo","Elapsed":0} +{"Time":"2026-10-20T22:20:06.030583418+02:00","Action":"output","Package":"fixture/failing/good","Output":"PASS\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.030962814+02:00","Action":"output","Package":"fixture/failing/bad","Output":"FAIL\tfixture/failing/bad\t0.002s\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.030974515+02:00","Action":"output","Package":"fixture/failing/good","Output":"ok \tfixture/failing/good\t0.002s\n"} +{"Time":"2026-10-20T22:20:06.030991305+02:00","Action":"pass","Package":"fixture/failing/good","Elapsed":0.002} +{"Time":"2026-10-20T22:20:06.030982975+02:00","Action":"fail","Package":"fixture/failing/bad","Elapsed":0.003} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/golist-external.json b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/golist-external.json new file mode 100644 index 00000000..189cb5cb --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/golist-external.json @@ -0,0 +1,13 @@ +{ + "ImportPath": "git.novox.be/novox/mesh-sdk/go/buildstep", + "Module": { + "Path": "git.novox.be/novox/mesh-sdk/go", + "Version": "v0.1.14-0.20261020200715-9e296295c05e", + "Time": "2026-10-20T20:07:15Z", + "Dir": "/home/jochen/go/pkg/mod/git.novox.be/novox/mesh-sdk/go@v0.1.14-0.20261020200715-9e296295c05e", + "GoMod": "/home/jochen/go/pkg/mod/cache/download/git.novox.be/novox/mesh-sdk/go/@v/v0.1.14-0.20261020200715-9e296295c05e.mod", + "GoVersion": "1.26.0", + "Sum": "h1:uf2o2A6dBOrISDUVw6U7+fv8ylwwF+UfRrGWKIQMFOE=", + "GoModSum": "h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=" + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/slow.json b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/slow.json new file mode 100644 index 00000000..1410ee59 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/slow.json @@ -0,0 +1,12 @@ +{"Time":"2026-10-20T22:20:06.21802535+02:00","Action":"start","Package":"fixture/slow"} +{"Time":"2026-10-20T22:20:06.222338257+02:00","Action":"run","Package":"fixture/slow","Test":"TestTakesAWhile"} +{"Time":"2026-10-20T22:20:06.222387677+02:00","Action":"output","Package":"fixture/slow","Test":"TestTakesAWhile","Output":"=== RUN TestTakesAWhile\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.282860204+02:00","Action":"output","Package":"fixture/slow","Test":"TestTakesAWhile","Output":"--- PASS: TestTakesAWhile (0.06s)\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.282917774+02:00","Action":"pass","Package":"fixture/slow","Test":"TestTakesAWhile","Elapsed":0.06} +{"Time":"2026-10-20T22:20:06.282937215+02:00","Action":"run","Package":"fixture/slow","Test":"TestQuick"} +{"Time":"2026-10-20T22:20:06.282943535+02:00","Action":"output","Package":"fixture/slow","Test":"TestQuick","Output":"=== RUN TestQuick\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.282955095+02:00","Action":"output","Package":"fixture/slow","Test":"TestQuick","Output":"--- PASS: TestQuick (0.00s)\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.282963365+02:00","Action":"pass","Package":"fixture/slow","Test":"TestQuick","Elapsed":0} +{"Time":"2026-10-20T22:20:06.282971185+02:00","Action":"output","Package":"fixture/slow","Output":"PASS\n","OutputType":"frame"} +{"Time":"2026-10-20T22:20:06.283463443+02:00","Action":"output","Package":"fixture/slow","Output":"ok \tfixture/slow\t0.063s\n"} +{"Time":"2026-10-20T22:20:06.283481213+02:00","Action":"pass","Package":"fixture/slow","Elapsed":0.065} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/tagged.json b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/tagged.json new file mode 100644 index 00000000..8a6e1b0f --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/tagged.json @@ -0,0 +1,3 @@ +{"Time":"2026-10-20T22:20:06.049418313+02:00","Action":"start","Package":"fixture/tagged"} +{"Time":"2026-10-20T22:20:06.049698258+02:00","Action":"output","Package":"fixture/tagged","Output":"? \tfixture/tagged\t[no test files]\n"} +{"Time":"2026-10-20T22:20:06.049730188+02:00","Action":"skip","Package":"fixture/tagged","Elapsed":0} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/toolchain-go-entry.txt b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/toolchain-go-entry.txt new file mode 100644 index 00000000..73a4664b --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/toolchain-go-entry.txt @@ -0,0 +1,37 @@ + Language: "go", + Base: "mesh-tools-go", + Artifact: "build", + // **The mesh's own components, and not modules.** The warning above this list — that every + // language is another implementation of the contracts modules share, so adding one commits + // to keeping N implementations in step — does not attach here. Go is how the host, the + // control plane and the builder are written, and none of them is a module in that sense: + // the host is what APPLIES modules. So there is no SDK obligation, and the reason this + // entry did not exist was that nothing needed to compile the mesh itself + // (novox/hq ADR 0142, and 04-ISSUES/142 where that is why nothing delivers the host). + // + // Static, because what a machine ends up holding is a file rather than a container, and a + // binary that needs a libc it did not bring is a delivery that works until a machine + // differs. Trimmed of its own paths for the same reason a version comes from where it sits + // rather than from the linker: two builds of one commit produce the same bytes. + // + // **And of one build source at two commits** (novox/hq ADR 0267 rule 6, issue 363): a module built + // from its repository's root is compiled with the clone's `.git` in view, and Go stamps the commit, + // its time and whether the tree was modified into the binary unless told not to — so a merge that + // changed only a README made a new digest of an unchanged program. + Compile: []string{ + "env", "CGO_ENABLED=0", "GOFLAGS=-trimpath -buildvcs=false", + "go", "build", + }, + // Stripped of symbols and debug info: what a machine holds is a file it runs, not one it + // debugs, and the difference measured 12.2MB against 8.5MB. + LinkerFlags: []string{"-s", "-w"}, + OutputFlag: "-o", + // Pointed at the package the artifact is built `from`, compiled whole. Go writes the binary + // into the output directory, named after the package — so the bundle a machine unpacks is a + // directory holding one executable, which is what the delivery mechanism expects + // (novox/hq ADR 0141). + Unit: UnitPackage, + // The mesh's own Go components read the system they were built for from this variable, and + // refuse to touch a machine without one. + SystemStamp: "main.builtFor", + }, diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/broken.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/broken.go new file mode 100644 index 00000000..02f0ddf3 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/broken.go @@ -0,0 +1,3 @@ +package broken + +func One() int { return "one" } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/broken_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/broken_test.go new file mode 100644 index 00000000..34eb0541 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/broken_test.go @@ -0,0 +1,5 @@ +package broken + +import "testing" + +func TestOne(t *testing.T) { One() } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/go.mod new file mode 100644 index 00000000..e0f4d228 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/go.mod @@ -0,0 +1,3 @@ +module fixture/broken + +go 1.25.0 diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/cmd/prog/main.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/cmd/prog/main.go new file mode 100644 index 00000000..4037a92a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/cmd/prog/main.go @@ -0,0 +1,9 @@ +// Command prog is what the build step of issue 1487 builds in its integration test. +package main + +import "fmt" + +// builtFor is stamped by the linker. +var builtFor string + +func main() { fmt.Println("prog built for", builtFor) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/go.mod new file mode 100644 index 00000000..00ca3e6d --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/go.mod @@ -0,0 +1,3 @@ +module fixture/build + +go 1.25.0 diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/module.json b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/module.json new file mode 100644 index 00000000..b5439527 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/module.json @@ -0,0 +1,17 @@ +{ + "module": "fixture-build", + "version": "1", + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/prog", + "binary": "prog-bin", + "loads": ["prog-bin"] + } + ] + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/a.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/a.go new file mode 100644 index 00000000..0f746116 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/a.go @@ -0,0 +1,4 @@ +package a + +// A is what a's tests check. +func A() int { return 1 } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/a_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/a_test.go new file mode 100644 index 00000000..0aaf337a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/a_test.go @@ -0,0 +1,15 @@ +package a + +import ( + "os" + "testing" + + "fixture/digest/b" +) + +func TestAReadsBAndItsTestdata(t *testing.T) { + raw, err := os.ReadFile("testdata/x.txt") + if err != nil || A()+b.B() != 3 || len(raw) == 0 { + t.Fatal("a") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/testdata/x.txt b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/testdata/x.txt new file mode 100644 index 00000000..587be6b4 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/testdata/x.txt @@ -0,0 +1 @@ +x diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/b/b.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/b/b.go new file mode 100644 index 00000000..690fd406 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/b/b.go @@ -0,0 +1,4 @@ +package b + +// B is reached by a's tests. +func B() int { return 2 } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/c.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/c.go new file mode 100644 index 00000000..467fd27e --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/c.go @@ -0,0 +1,6 @@ +package c + +import _ "embed" + +//go:embed note.txt +var Note string diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/c_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/c_test.go new file mode 100644 index 00000000..4ab9f706 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/c_test.go @@ -0,0 +1,9 @@ +package c + +import "testing" + +func TestCHasANote(t *testing.T) { + if Note == "" { + t.Fatal("c") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/note.txt b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/note.txt new file mode 100644 index 00000000..519dd581 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/note.txt @@ -0,0 +1 @@ +note diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/d/d.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/d/d.go new file mode 100644 index 00000000..9cd66d1a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/d/d.go @@ -0,0 +1,4 @@ +package d + +// D is reached by no test. +func D() int { return 4 } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/go.mod new file mode 100644 index 00000000..2659cd0a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/go.mod @@ -0,0 +1,3 @@ +module fixture/digest + +go 1.25.0 diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/empty/empty.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/empty/empty.go new file mode 100644 index 00000000..d0f06191 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/empty/empty.go @@ -0,0 +1,3 @@ +package empty + +func Nothing() {} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/empty/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/empty/go.mod new file mode 100644 index 00000000..ac1acaba --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/empty/go.mod @@ -0,0 +1,3 @@ +module fixture/empty + +go 1.25.0 diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/bad/bad.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/bad/bad.go new file mode 100644 index 00000000..526a6282 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/bad/bad.go @@ -0,0 +1,3 @@ +package bad + +func Three() int { return 3 } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/bad/bad_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/bad/bad_test.go new file mode 100644 index 00000000..b0d25fa0 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/bad/bad_test.go @@ -0,0 +1,9 @@ +package bad + +import "testing" + +func TestThreeIsFour(t *testing.T) { + if Three() != 4 { + t.Fatalf("Three() = %d, want 4", Three()) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/go.mod new file mode 100644 index 00000000..10b202f0 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/go.mod @@ -0,0 +1,3 @@ +module fixture/failing + +go 1.25.0 diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/good/good.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/good/good.go new file mode 100644 index 00000000..6022f6a1 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/good/good.go @@ -0,0 +1,3 @@ +package good + +func Two() int { return 2 } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/good/good_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/good/good_test.go new file mode 100644 index 00000000..c6392073 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/good/good_test.go @@ -0,0 +1,9 @@ +package good + +import "testing" + +func TestTwo(t *testing.T) { + if Two() != 2 { + t.Fatal("two") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/go.mod new file mode 100644 index 00000000..7f95cab9 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/go.mod @@ -0,0 +1,3 @@ +module fixture/slow + +go 1.25.0 diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/slow.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/slow.go new file mode 100644 index 00000000..10b49fa5 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/slow.go @@ -0,0 +1,3 @@ +package slow + +func Zero() int { return 0 } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/slow_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/slow_test.go new file mode 100644 index 00000000..39a56be8 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/slow_test.go @@ -0,0 +1,16 @@ +package slow + +import ( + "testing" + "time" +) + +func TestTakesAWhile(t *testing.T) { + time.Sleep(60 * time.Millisecond) +} + +func TestQuick(t *testing.T) { + if Zero() != 0 { + t.Fatal("zero") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/go.mod new file mode 100644 index 00000000..71f77819 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/go.mod @@ -0,0 +1,3 @@ +module fixture/store + +go 1.25.0 diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/store.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/store.go new file mode 100644 index 00000000..cff0dca8 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/store.go @@ -0,0 +1,4 @@ +package store + +// Name is what the integration test below finds a store and a bus for. +const Name = "store" diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/store_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/store_integration_test.go new file mode 100644 index 00000000..9c3e6a40 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/store_integration_test.go @@ -0,0 +1,37 @@ +//go:build integration + +package store + +import ( + "bufio" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// The fixture of issue 1486: the store and the bus a run gives its tests are there, reached as a test reaches them. +func TestTheStoreAndTheBusAreThere(t *testing.T) { + u, err := url.Parse(os.Getenv("MESH_TEST_POSTGRES")) + if err != nil || u.Query().Get("host") == "" { + t.Fatalf("MESH_TEST_POSTGRES = %q", os.Getenv("MESH_TEST_POSTGRES")) + } + pg, err := net.DialTimeout("unix", filepath.Join(u.Query().Get("host"), ".s.PGSQL.5432"), 5*time.Second) + if err != nil { + t.Fatalf("the store's socket: %v", err) + } + pg.Close() + + bus, err := net.DialTimeout("tcp", strings.TrimPrefix(os.Getenv("MESH_TEST_NATS"), "nats://"), 5*time.Second) + if err != nil { + t.Fatalf("the bus: %v", err) + } + defer bus.Close() + line, err := bufio.NewReader(bus).ReadString('\n') + if err != nil || !strings.HasPrefix(line, "INFO ") || !strings.Contains(line, `"jetstream":true`) { + t.Fatalf("the bus said %q (%v), want an INFO line with JetStream on", line, err) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/go.mod new file mode 100644 index 00000000..4be2fcd2 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/go.mod @@ -0,0 +1,3 @@ +module fixture/tagged + +go 1.25.0 diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/tagged.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/tagged.go new file mode 100644 index 00000000..9be2b802 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/tagged.go @@ -0,0 +1,3 @@ +package tagged + +func One() int { return 1 } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/tagged_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/tagged_integration_test.go new file mode 100644 index 00000000..917c1765 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/tagged_integration_test.go @@ -0,0 +1,11 @@ +//go:build integration + +package tagged + +import "testing" + +func TestNeedsAStore(t *testing.T) { + if One() != 1 { + t.Fatal("one") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/tools.go b/modules/mesh-build-go/cmd/mesh-build-go/tools.go new file mode 100644 index 00000000..91c2fd1a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/tools.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The tools this module serves. A tool is named as the module is: build_go_ and its verb (a tool name has no dot). +const ( + StatusTool = "build_go_status" + HealthTool = "build_go_health" +) + +// StepTool is one step tool of the module: fetch, test, digest or build. Long is a step that answers at once and +// ends on its key (ADR 0469 §2); the others answer in their call. Both run behind the toolchain guard. +type StepTool struct { + Name string + Description string + Input map[string]any + Long bool + Run func(ctx context.Context, args map[string]any) (any, error) +} + +// StepTools are the module's step tools. Each step's issue adds its own. +func (m *Module) StepTools() []StepTool { + return []StepTool{m.fetchTool(), m.testTool(), m.digestTool(), m.buildTool()} +} + +// Tools is every tool the module serves: its step tools behind the guard, its status and its health. +func (m *Module) Tools(steps []StepTool) []stdio.Tool { + tools := []stdio.Tool{} + for _, s := range steps { + t := stdio.Tool{Name: s.Name, Description: s.Description, Input: s.Input} + if s.Long { + t.Run = m.Long(s.Name, s.Run) + } else { + t.Run = m.Short(s.Run) + } + tools = append(tools, t) + } + return append(tools, + stdio.Tool{Name: StatusTool, + Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " + + "the recipe version, the postgres major version and the nats-server version the integration level uses, and " + + "why it refuses to build if it does. Read-only.", + Run: func(map[string]any) (any, error) { return m.Status(m.Ctx), nil }}, + stdio.Tool{Name: HealthTool, + Description: "How the Go build module is: unhealthy while the Go installed on this machine is not the version its " + + "recipe builds with, saying both versions; healthy otherwise. Read-only.", + Run: func(map[string]any) (any, error) { return m.Health(m.Ctx), nil }}, + ) +} diff --git a/modules/mesh-build-go/files/forge b/modules/mesh-build-go/files/forge new file mode 100644 index 00000000..6bbf4e20 --- /dev/null +++ b/modules/mesh-build-go/files/forge @@ -0,0 +1,7 @@ +Host ${bound:forge-account:ssh_alias} + HostName ${bound:forge-account:ssh_host} + Port ${bound:forge-account:ssh_port} + HostKeyAlias ${bound:forge-account:ssh_alias} + User git + IdentityFile ~/.ssh/mesh_ed25519 + IdentitiesOnly yes diff --git a/modules/mesh-build-go/files/gitconfig b/modules/mesh-build-go/files/gitconfig new file mode 100644 index 00000000..05252302 --- /dev/null +++ b/modules/mesh-build-go/files/gitconfig @@ -0,0 +1,2 @@ +[url "ssh://${bound:forge-account:ssh_alias}/"] + insteadOf = ${bound:forge-account:https_base} diff --git a/modules/mesh-build-go/files/known_hosts b/modules/mesh-build-go/files/known_hosts new file mode 100644 index 00000000..e0df6d81 --- /dev/null +++ b/modules/mesh-build-go/files/known_hosts @@ -0,0 +1 @@ +${bound:forge-account:ssh_alias} ${bound:forge-account:host_key} diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod new file mode 100644 index 00000000..464af931 --- /dev/null +++ b/modules/mesh-build-go/go.mod @@ -0,0 +1,21 @@ +module meshbuildgo + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d + github.com/nats-io/nats-server/v2 v2.11.17 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum new file mode 100644 index 00000000..4ef33264 --- /dev/null +++ b/modules/mesh-build-go/go.sum @@ -0,0 +1,31 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020200715-9e296295c05e h1:uf2o2A6dBOrISDUVw6U7+fv8ylwwF+UfRrGWKIQMFOE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020200715-9e296295c05e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020203945-1572e50e0ef0 h1:mr9HyARwzeZT7FxFHv1Xc2z7QErvDZ/F/pBMIpjGSd0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020203945-1572e50e0ef0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d h1:mmOCJphjQN15jNVh+sMQqnmJ3UiYltiuQRg/zw3wFtw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json new file mode 100644 index 00000000..85944415 --- /dev/null +++ b/modules/mesh-build-go/module.json @@ -0,0 +1,148 @@ +{ + "module": "mesh-build-go", + "version": "1", + "runs-as": "mesh-build-go", + "capabilities": [ + "package-manager", + "user-namespaces" + ], + "state": [ + "toolchains", + "keys", + "steps" + ], + "tools": [ + "build_go_fetch", + "build_go_test", + "build_go_digest", + "build_go_build", + "build_go_status", + "build_go_health" + ], + "streams": [ + { + "name": "toolchains-go", + "events": ["toolchain.**"] + } + ], + "requires": [ + "forge-account" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-build-go", + "runtime-health": { + "kind": "tool", + "tool": "build_go_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + }, + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-build-go", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-build-go" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-build-go", + "place": ".", + "unknown": "program" + }, + { + "id": "work", + "type": "directory", + "path": "/var/lib/mesh-build-go-work", + "mode": "0700", + "owner": "mesh-build-go", + "unknown": "program" + }, + { + "id": "cache", + "type": "directory", + "path": "/var/lib/mesh-build-go-cache", + "mode": "0700", + "owner": "mesh-build-go", + "unknown": "program" + }, + { + "id": "ssh-dir", + "type": "directory", + "path": "${dir:state}/.ssh", + "mode": "0700", + "owner": "mesh-build-go", + "unknown": "program" + }, + { + "id": "ssh-config", + "type": "file", + "path": "${dir:ssh-dir}/config", + "owner": "mesh-build-go", + "mode": "0600", + "template": "files/forge" + }, + { + "id": "known-hosts", + "type": "file", + "path": "${dir:ssh-dir}/known_hosts", + "owner": "mesh-build-go", + "mode": "0644", + "template": "files/known_hosts" + }, + { + "id": "git-config", + "type": "file", + "path": "${dir:state}/.config/git/config", + "owner": "mesh-build-go", + "mode": "0644", + "template": "files/gitconfig" + }, + { + "id": "go", + "type": "package", + "package": "go" + }, + { + "id": "git", + "type": "package", + "package": "git" + }, + { + "id": "openssh", + "type": "package", + "package": "openssh" + }, + { + "id": "postgresql", + "type": "package", + "package": "postgresql" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-build-go", + "binary": "mesh-build-go", + "loads": [ + "mesh-build-go" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-build-node/README.md b/modules/mesh-build-node/README.md new file mode 100644 index 00000000..19c16e8e --- /dev/null +++ b/modules/mesh-build-node/README.md @@ -0,0 +1,67 @@ +# mesh-build-node — the Node and TypeScript build module + +Design 47, release R4c-3 (novox/hq ADR 0405, ADR 0466 §1, ADR 0469 §1 and §4). The module that builds and checks +TypeScript code on a build machine, as an account of its own (`runs-as`, no login shell, home at mode 0700), with the +packages it declares (`nodejs`, `npm`, `git`, `openssh`) and the capability `user-namespaces`. The bundle is Go +(`cmd/mesh-build-node`). It claims no seat: it is assigned to the build machines (ADR 0469 §6) and runs dark until the +builder uses it. Its forge key and ssh files are the same as mesh-build-go's (issue 1484). + +## The recipe + +`recipe.go` names the one Node version it builds with (`RecipeNode`, `v26.10.0`, as `node --version` prints it) and the +recipe's own version (`RecipeSteps`). The recipe version is a hash of those two and of the bytes of the lock file +`toolchain/package-lock.json`, so one changed byte of the lock, another Node or another recipe moves it. + +## The toolchain directory + +`toolchain/package.json` and its committed `package-lock.json` name typescript 5.9.3, esbuild 0.25.12, @types/node 22.20.1 (the types tsc checks node imports against), +@novox/mesh-sdk 0.1.14 and nats 2.29.3. The mesh unpacks both, as the archive artifact `toolchain`, into `/toolchain/` (resource `toolchain-dir`; a +manifest names no host, and the lock does, so it is not a template); the program reads the lock from there. When `/toolchain/` is missing +the module runs `npm ci --ignore-scripts` there once (a directory without its `.installed` mark counts as missing). + +**How the registry is reached** (the choice of issue 1488): the lock holds @novox/mesh-sdk at the forge's public +address, and npm 12 installs a locked tarball only from the registry it is configured with. The module therefore +requires `npm-package-registry` (read access, provided by gitea) and, like the lab module, takes its binding and sealed +secret as files in its home. The install gets a private npm config file with the `@novox` scope at the binding's +`public-root` (else scheme, address, port and npm path) and the bound account's credential, and removes it afterwards; +every other package resolves from the public registry. This is the way today's build agent names it. + +The runtime tree for bundles with externals (the pruned sdk and nats today in `/app/runtime`) is not installed +separately: sdk and nats are in this directory's `node_modules`, and the build step (issue 1490) takes them from there. + +## The guard and the health + +At start, every ten minutes, before every step and at every health look, the module reads the lock again and the +installed Node (`node --version`) and writes it with the recipe version to `.typescript` of its `toolchains` +state. A machine whose Node is not the recipe's refuses every step, the step ending `refused` on its key in `steps` with +both versions named, and `build_node_health` (the module's `runtime-health`) answers unhealthy with the same text, so +the controller raises the machine's condition. A step runs only after the toolchain directory is installed. + +## Tools + +`build_node_status` and `build_node_health` answer in their call. The step tools run behind the guard and the +toolchain install: a long step answers at once with its id and ends on `.` of `steps` (ADR 0469 §2). + +- `build_node_fetch` {repository, commit, ref} (issue 1489): the commit over ssh as the module's account + (`buildstep.FetchCommit`, remote `ssh://mesh-forge//.git`) into `//tree`, then for each + TypeScript module (a module.json with a typescript artifact) the packages its package.json names beyond the SDK, + `typescript` and `@types/*` (the toolchain carries those): none installs nothing; some need a package-lock.json, and + are installed as the build agent does (scratch copy without the SDK, + devDependencies and scripts; `npm ci --omit=dev --omit=peer --ignore-scripts --no-bin-links`; copies of the SDK + removed). The mesh's registry reaches `npm ci` through a private config file (`--userconfig`), so + other `@novox/` packages install. A module with packages and no lock does not fail the fetch: its result lists it + under `refused` with the reason, and a build of that bundle fails naming it (issue 1459). Every command runs through `forgekey.ExecRunnerClean(0)` with PATH, HOME, the locale and its own + words, nothing of this process's environment. Workspaces older than a day are removed at each fetch. +- `build_node_test` {workspace, module, level} (issue 1491): the unit level is the type-check (the toolchain's tsc, the + package's `tsconfig.json` or the build agent's flags over its .ts files) and the `test` script; the integration level + is the `test:integration` script, the type-check being its setup. Bounds (the Go module's; ADR 0445 names none for + TypeScript): unit 5 s, integration 30 s setup and 60 s running. The test script finds `tsc` and `esbuild` on its PATH through `//bin` (shell scripts calling the toolchain's). A level over its bound says `over_bound` and passes; + a failed one finishes the step with `passed` false and the output (the step fails only when the tests could not be run). Neither script nor .ts files: no tests. The toolchain makes no bin + links, so a script runs `node` and files, not `tsc` by name. +- `build_node_build` {workspace, module, artifact} (issue 1490): the build agent's tsc and esbuild lines + (mesh-controller internal/builder, cited in `build.go`), the launchers, `package.json`, `buildstep.Pack`, and the + digest. No network. The build runs in a staging copy of the module at `//stage-/modules/module`, with the + toolchain's packages (not typescript or esbuild) copied to `stage-/node_modules`: the build agent's depth + (`/app/modules/module`, `/app/node_modules`), so esbuild's `../../node_modules/...` path comments, and the bundle's + digest, equal its. Copies, not links, because the mesh makes no symlink. A bundle that keeps an external gets them in its own `node_modules`. + The packed archive is written beside the tree as `.tar.gz`. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build.go b/modules/mesh-build-node/cmd/mesh-build-node/build.go new file mode 100644 index 00000000..a5f9519a --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/build.go @@ -0,0 +1,283 @@ +package main + +// The build step (novox/hq ADR 0405 §3 and §7, ADR 0469 §4; issue 1490): a TypeScript bundle of a fetched workspace +// compiled with the toolchain's tsc, bundled with its esbuild, given its launchers, packed with mesh-sdk's Pack and +// ended with the archive's digest. No network. The commands and flags are the build agent's, copied from +// mesh-controller origin/main internal/builder: toolchain.go:143-147 (tsc), builder.go:1637-1705 (compile), +// :1926-1956 (writeLaunchers), :1971-2039 (bundled), :1735-1747 (sourcesFor). Where they ran in the toolchain image +// they run here from the module's directory with the toolchain's real paths, and tsc and esbuild find the SDK in the +// workspace's node_modules (workspace.go). + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// launcherSuffix is what an entrypoint's launcher is called beside it: index.js is started as index.serve.mjs +// (novox/hq ADR 0193). +const launcherSuffix = ".serve.mjs" + +// Out is where a bundle's compiled files go in the module's directory (builder toolchain.go:118). +func Out(artifact string) string { return ".mesh-build/" + artifact } + +// banner is esbuild's createRequire shim: CommonJS inlined into an ES module still finds `require`. +const banner = "--banner:js=import { createRequire as __meshRequire } from 'node:module'; const require = __meshRequire(import.meta.url);" + +// artifact is the part of a module.json artifact the build reads. +type artifact struct { + Name string `json:"name"` + Language string `json:"language"` + Entrypoints []string `json:"entrypoints"` + External []string `json:"external"` +} + +// BuildResult is what a passed build step says. +type BuildResult struct { + Workspace string `json:"workspace"` + Module string `json:"module"` + Artifact string `json:"artifact"` + // Digest is "sha256:" and the hex of the archive's bytes (buildstep.Pack). + Digest string `json:"digest"` + Bytes int `json:"bytes"` + // Archive is the packed file, in the workspace beside its tree. + Archive string `json:"archive"` + Files []string `json:"files"` +} + +func (m *Module) buildTool() StepTool { + return StepTool{ + Name: "build_node_build", + Description: "Build a TypeScript bundle of a fetched workspace: compile it with the toolchain's tsc, bundle each " + + "entrypoint with its esbuild, write the launchers and package.json, pack the bundle and answer its digest. " + + "Answers a step id at once; the step ends on its key in the steps state with the digest. No network.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id a fetch step answered"}, + "module": map[string]any{"type": "string", "description": "the module's directory in the workspace, '.' when it is the root"}, + "artifact": map[string]any{"type": "string", "description": "the name of the bundle in the module's module.json"}, + }, + "required": []string{"workspace", "artifact"}, + }, + Long: true, + Run: m.build, + } +} + +func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { + workspace, _ := args["workspace"].(string) + module, _ := args["module"].(string) + name, _ := args["artifact"].(string) + root, dir, err := m.moduleDirOf(workspace, module) + if err != nil { + return nil, err + } + a, err := readArtifact(dir, name) + if err != nil { + return nil, err + } + if why, err := lockProblem(dir, module); err != nil || why != "" { + if err == nil { + err = fmt.Errorf("%s was refused by the fetch: %s", name, why) + } + return nil, err + } + // Built at the build agent's depth (builder.go: the module at /app/modules/module, the packages at /app/node_modules), + // so the path comments esbuild writes, `../../node_modules/...`, and with them the bundle's bytes equal its. + stage := filepath.Join(root, "stage-"+a.Name) + if err := os.RemoveAll(stage); err != nil { + return nil, err + } + src := dir + dir = filepath.Join(stage, "modules", "module") + if err := copyTree(src, dir); err != nil { + return nil, fmt.Errorf("staging the module: %w", err) + } + if err := copyPackages(filepath.Join(m.toolchainDir(), "node_modules"), filepath.Join(stage, "node_modules")); err != nil { + return nil, fmt.Errorf("providing the toolchain's packages to the build: %w", err) + } + out, final := Out(a.Name), Out(a.Name)+".bundled" + if err := os.MkdirAll(filepath.Join(dir, out), 0o755); err != nil { + return nil, err + } + // tsc: builder.go:1637-1705 with toolchain.go:143-147's flags, rooted at the module so an entrypoint lands where + // it is named. + tsc := []string{m.tsc(), "--module", "NodeNext", "--moduleResolution", "NodeNext", "--target", "ES2022", + "--rootDir", ".", "--outDir", out} + tsc = append(tsc, sourcesFor(a.Entrypoints, out, ".ts")...) + if why := ranFailed(m.runIn(ctx, dir, nil, "node", tsc...)); why != "" { + return nil, fmt.Errorf("tsc for %s: %s", a.Name, why) + } + launchers, err := writeLaunchers(filepath.Join(dir, out), a) + if err != nil { + return nil, err + } + if err := m.bundle(ctx, dir, a, launchers); err != nil { + return nil, err + } + bundleDir := filepath.Join(dir, final) + if len(a.External) > 0 { + if err := copyPackages(filepath.Join(m.toolchainDir(), "node_modules"), filepath.Join(bundleDir, "node_modules")); err != nil { + return nil, fmt.Errorf("copying the packages %s keeps external: %w", a.Name, err) + } + } + archive, digest, err := buildstep.Pack(bundleDir) + if err != nil { + return nil, fmt.Errorf("packing %s: %w", a.Name, err) + } + file := filepath.Join(root, a.Name+".tar.gz") + if err := os.WriteFile(file, archive, 0o600); err != nil { + return nil, err + } + files := []string{} + _ = filepath.WalkDir(bundleDir, func(p string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() { + rel, _ := filepath.Rel(bundleDir, p) + files = append(files, filepath.ToSlash(rel)) + } + return nil + }) + sort.Strings(files) + return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive), Archive: file, Files: files}, nil +} + +// bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is +// bundled in place under its own name, a launcher keeps its name with the extension .mjs. +func (m *Module) bundle(ctx context.Context, dir string, a artifact, launchers map[string]string) error { + out, final := Out(a.Name), Out(a.Name)+".bundled" + if err := os.RemoveAll(filepath.Join(dir, final)); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(dir, final), 0o755); err != nil { + return err + } + common := []string{"--bundle", "--platform=node", "--format=esm", "--target=node22", + "--outbase=" + out, "--outdir=" + final, "--log-level=warning", banner} + for _, x := range a.External { + common = append(common, "--external:"+x) + } + var plain, launch []string + for _, e := range a.Entrypoints { + if strings.HasSuffix(e, ".js") { + plain = append(plain, out+"/"+e) + } + } + for _, l := range sortedValues(launchers) { + launch = append(launch, out+"/"+l) + } + step := func(entries []string, extra ...string) error { + if len(entries) == 0 { + return nil + } + args := append(append(append([]string{}, entries...), common...), extra...) + if why := ranFailed(m.runIn(ctx, dir, nil, m.esbuild(), args...)); why != "" { + return fmt.Errorf("esbuild for %s: %s", a.Name, why) + } + return nil + } + if err := step(plain); err != nil { + return err + } + if err := step(launch, "--out-extension:.js=.mjs"); err != nil { + return err + } + // Plain `.js` output is an ES module; said once. + if err := os.WriteFile(filepath.Join(dir, final, "package.json"), []byte(`{"type":"module","private":true}`+"\n"), 0o644); err != nil { + return err + } + for _, l := range launchers { + path := filepath.Join(dir, final, filepath.FromSlash(l)) + if _, err := os.Stat(path); err == nil { + if err := os.Chmod(path, 0o755); err != nil { + return err + } + } + } + return nil +} + +// writeLaunchers writes, beside every entrypoint, an executable that imports the entrypoint and serves what it +// registered over MCP on stdio (builder.go:1926-1956; novox/hq ADR 0193). Its answer is each entrypoint's launcher, +// relative to the bundle's root. +func writeLaunchers(root string, a artifact) (map[string]string, error) { + out := map[string]string{} + for _, entry := range a.Entrypoints { + if !strings.HasSuffix(entry, ".js") { + continue + } + launcher := strings.TrimSuffix(entry, ".js") + launcherSuffix + body := "#!/usr/bin/env node\n" + + "// Written by the mesh's builder (novox/hq ADR 0193): serve what " + entry + " registers,\n" + + "// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.\n" + + "import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\n" + + "await import(\"./" + filepath.Base(entry) + "\");\n" + + "await serveRegisteredOverStdio();\n" + path := filepath.Join(root, filepath.FromSlash(launcher)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, err + } + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + return nil, err + } + if err := os.Chmod(path, 0o755); err != nil { + return nil, err + } + out[entry] = launcher + } + return out, nil +} + +// sourcesFor turns compiled entrypoints back into what to compile (builder.go:1735-1747). +func sourcesFor(entrypoints []string, out, ext string) []string { + sources := make([]string, 0, len(entrypoints)) + for _, e := range entrypoints { + at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/") + sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+ext) + } + return sources +} + +func sortedValues(m map[string]string) []string { + out := make([]string, 0, len(m)) + for _, v := range m { + out = append(out, v) + } + sort.Strings(out) + return out +} + +// readArtifact reads the named artifact of the module.json in dir, which must be a TypeScript bundle with entrypoints. +func readArtifact(dir, name string) (artifact, error) { + raw, err := os.ReadFile(filepath.Join(dir, "module.json")) + if err != nil { + return artifact{}, fmt.Errorf("the module has no module.json: %w", err) + } + var m struct { + Build struct { + Artifacts []artifact `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + return artifact{}, fmt.Errorf("module.json is not JSON: %w", err) + } + for _, a := range m.Build.Artifacts { + if a.Name != name { + continue + } + if a.Language != "typescript" { + return artifact{}, fmt.Errorf("artifact %q is %q, and this module builds typescript", name, a.Language) + } + if len(a.Entrypoints) == 0 { + return artifact{}, fmt.Errorf("artifact %q names no entrypoints", name) + } + return a, nil + } + return artifact{}, fmt.Errorf("module.json declares no artifact %q", name) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build_integration_test.go b/modules/mesh-build-node/cmd/mesh-build-node/build_integration_test.go new file mode 100644 index 00000000..a7faa3d6 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/build_integration_test.go @@ -0,0 +1,91 @@ +//go:build integration + +package main + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "slices" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// realMachine installs the real toolchain from the module's own lock (npm ci --ignore-scripts, from the mesh's +// registry for @novox, which reads without a credential) and makes two workspaces holding one fixture bundle. +func realMachine(t *testing.T) (*Module, string) { + t.Helper() + for _, p := range []string{"node", "npm"} { + if _, err := exec.LookPath(p); err != nil { + t.Skipf("NOT RUN: no %s installed where this check runs (the mesh's Go toolchain image holds none); proven live on a build node", p) + } + } + home, cache, work := t.TempDir(), t.TempDir(), t.TempDir() + for _, n := range []string{"package.json", "package-lock.json"} { + raw, err := os.ReadFile(filepath.Join(moduleDir, "toolchain", n)) + if err != nil { + t.Fatal(err) + } + write(t, filepath.Join(home, "toolchain", n), string(raw), 0o644) + } + lock, err := ReadRecipe(filepath.Join(home, "toolchain", "package-lock.json")) + if err != nil { + t.Fatal(err) + } + run := forgekey.ExecRunnerFor(0) + m := &Module{Node: "ace", Recipe: lock, Work: work, Run: run, Ctx: context.Background(), + Toolchain: &Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: run, Registry: func() (Registry, error) { + return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil + }}} + if err := m.Toolchain.Ensure(context.Background(), m.Recipe.Version()); err != nil { + t.Fatal(err) + } + for _, id := range []string{"w1", "w2"} { + for p, body := range map[string]string{ + "modules/m/module.json": `{"module":"m","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`, + "modules/m/package.json": `{"name":"m","type":"module","scripts":{"test":"node -e \"process.exit(0)\""}}`, + "modules/m/tools/index.ts": "import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\nexport const f: () => Promise = serveRegisteredOverStdio as () => Promise;\n", + } { + write(t, filepath.Join(work, id, "tree", filepath.FromSlash(p)), body, 0o644) + } + } + return m, work +} + +// 1490/AC2: a fixture TypeScript bundle importing @novox/mesh-sdk, built twice by the real tsc and esbuild, has equal +// digests, and its archive holds the bundled .js, the .serve.mjs launcher and package.json. +func TestAFixtureBundleBuiltTwiceHasOneDigest(t *testing.T) { + m, work := realMachine(t) + var digests []string + for _, id := range []string{"w1", "w2"} { + out, err := m.build(context.Background(), map[string]any{"workspace": id, "module": "modules/m", "artifact": "tools"}) + if err != nil { + t.Fatal(err) + } + r := out.(BuildResult) + digests = append(digests, r.Digest) + if want := []string{"package.json", "tools/index.js", "tools/index.serve.mjs"}; !slices.Equal(r.Files, want) { + t.Errorf("files = %v, want %v", r.Files, want) + } + } + if digests[0] != digests[1] { + t.Errorf("digests differ: %v", digests) + } + if links := symlinksUnder(t, work); len(links) != 0 { + t.Errorf("symlinks in the workspaces: %v", links) + } +} + +// 1491: the real type-check and test script of the same fixture pass the unit level. +func TestTheFixturePassesTheUnitLevel(t *testing.T) { + m, _ := realMachine(t) + out, err := m.test(context.Background(), map[string]any{"workspace": "w1", "module": "modules/m", "level": "unit"}) + if err != nil { + t.Fatal(err) + } + if r := out.(*TestResult); !r.Passed || r.NoTests || len(r.Ran) != 2 { + t.Errorf("result = %+v", r) + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build_test.go b/modules/mesh-build-node/cmd/mesh-build-node/build_test.go new file mode 100644 index 00000000..e59aefb2 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/build_test.go @@ -0,0 +1,256 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +const bundleManifest = `{"module":"m","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript",` + + `"entrypoints":["tools/index.js"],"loads":["tools/index.js"]%s}]}}` + +// esbuildEffect stands in for esbuild's output: for each entry `.mesh-build//.js` it writes the bundled file +// under --outdir, named as esbuild names it (the input without its extension, then .js or .mjs when --out-extension:.js=.mjs is given). +func esbuildEffect(t *testing.T) func(c call) { + return func(c call) { + if filepath.Base(c.name) != "esbuild" { + return + } + var outdir, outbase, ext string + var entries []string + for _, a := range c.args { + switch { + case strings.HasPrefix(a, "--outdir="): + outdir = strings.TrimPrefix(a, "--outdir=") + case strings.HasPrefix(a, "--outbase="): + outbase = strings.TrimPrefix(a, "--outbase=") + case a == "--out-extension:.js=.mjs": + ext = ".mjs" + case !strings.HasPrefix(a, "--"): + entries = append(entries, a) + } + } + for _, e := range entries { + rel := strings.TrimPrefix(strings.TrimPrefix(e, outbase), "/") + // esbuild names an output after its input without the extension, and adds .js or the one asked for. + rel = strings.TrimSuffix(rel, filepath.Ext(rel)) + if ext != "" { + rel += ext + } else { + rel += ".js" + } + write(t, filepath.Join(c.dir, outdir, rel), "bundled "+rel, 0o644) + } + } +} + +func buildArgs() map[string]any { + return map[string]any{"workspace": "w1", "module": "modules/m", "artifact": "tools"} +} + +func builtMachine(t *testing.T, extra string) (*Module, *fakeMachine, string) { + m, f, work := newMachine(t) + workspaceWith(t, work, map[string]string{"modules/m/module.json": strings.Replace(bundleManifest, "%s", extra, 1), + "modules/m/tools/index.ts": "export {}"}) + f.effect = esbuildEffect(t) + return m, f, work +} + +// 1490/AC1: tsc and esbuild run with exactly the build agent's flags. The fixture is copied from mesh-controller +// origin/main: internal/builder/toolchain.go:143-147 (tsc: --module NodeNext --moduleResolution NodeNext --target +// ES2022 --rootDir .), builder.go:1637-1705 (--outDir .mesh-build/ and the sources) and :1971-2039 (esbuild: +// --bundle --platform=node --format=esm --target=node22 --outbase --outdir --log-level=warning, the createRequire +// banner, --out-extension:.js=.mjs for the launchers, --external:x). Where the build agent ran them in the toolchain +// image they run here with the toolchain's real paths, from the module's directory (env -C). +func TestTscAndEsbuildRunWithTheBuildAgentsFlags(t *testing.T) { + m, f, work := builtMachine(t, `,"external":["pg"]`) + if _, err := m.build(context.Background(), buildArgs()); err != nil { + t.Fatal(err) + } + dir := filepath.Join(work, "w1", "stage-tools", "modules", "module") + nm := filepath.Join(m.toolchainDir(), "node_modules") + banner := "--banner:js=import { createRequire as __meshRequire } from 'node:module'; const require = __meshRequire(import.meta.url);" + common := []string{"--bundle", "--platform=node", "--format=esm", "--target=node22", + "--outbase=.mesh-build/tools", "--outdir=.mesh-build/tools.bundled", "--log-level=warning", banner, "--external:pg"} + want := []string{ + "node " + nm + "/typescript/bin/tsc --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . --outDir .mesh-build/tools tools/index.ts", + nm + "/esbuild/bin/esbuild .mesh-build/tools/tools/index.js " + strings.Join(common, " "), + nm + "/esbuild/bin/esbuild .mesh-build/tools/tools/index.serve.mjs " + strings.Join(common, " ") + " --out-extension:.js=.mjs", + } + if len(f.calls) != len(want) { + t.Fatalf("%d commands ran, want %d: %v", len(f.calls), len(want), f.calls) + } + for i, c := range f.calls { + if c.dir != dir || joined(c) != want[i] { + t.Errorf("command %d: in %s ran\n%s\nwant in %s\n%s", i, c.dir, joined(c), dir, want[i]) + } + } +} + +func readArchive(t *testing.T, path string) map[string]os.FileMode { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + zr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + tr := tar.NewReader(zr) + out := map[string]os.FileMode{} + for { + h, err := tr.Next() + if err == io.EOF { + return out + } + if err != nil { + t.Fatal(err) + } + out[h.Name] = os.FileMode(h.Mode) + } +} + +// 1490: the step packs the bundle: the archive holds the bundled .js, the .serve.mjs launcher (executable) and the +// package.json that makes it an ES module, and the result is the archive's digest, the same for the same build. +func TestTheBundleIsPackedWithItsLauncherAndPackageJson(t *testing.T) { + m, _, _ := builtMachine(t, ``) + out, err := m.build(context.Background(), buildArgs()) + if err != nil { + t.Fatal(err) + } + r := out.(BuildResult) + if !strings.HasPrefix(r.Digest, "sha256:") || r.Bytes == 0 { + t.Errorf("result = %+v", r) + } + if want := []string{"package.json", "tools/index.js", "tools/index.serve.mjs"}; !slices.Equal(r.Files, want) { + t.Errorf("files = %v, want %v", r.Files, want) + } + modes := readArchive(t, r.Archive) + if len(modes) != 3 || modes["tools/index.serve.mjs"]&0o111 == 0 { + t.Errorf("archive = %v, want the three files with the launcher executable", modes) + } + again, err := m.build(context.Background(), buildArgs()) + if err != nil || again.(BuildResult).Digest != r.Digest { + t.Errorf("a second build: %v, digest %v, want %s", err, again, r.Digest) + } + launcher, _ := os.ReadFile(filepath.Join(filepath.Dir(r.Archive), "stage-tools", "modules", "module", ".mesh-build", "tools", "tools", "index.serve.mjs")) + if !strings.Contains(string(launcher), `import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";`) || !strings.Contains(string(launcher), `await import("./index.js");`) { + t.Errorf("launcher = %q", launcher) + } +} + +// 1490/AC3: no symlink is made anywhere in the workspace, with externals (whose packages are copied) or without. +func TestNoSymlinkIsMadeInTheWorkspace(t *testing.T) { + for _, extra := range []string{``, `,"external":["nats"]`} { + m, _, work := builtMachine(t, extra) + if _, err := m.build(context.Background(), buildArgs()); err != nil { + t.Fatal(err) + } + if links := symlinksUnder(t, filepath.Join(work, "w1")); len(links) != 0 { + t.Errorf("symlinks in the workspace: %v", links) + } + } +} + +// 1490: the bundle's SDK comes from the staging directory's node_modules, two levels above the module, and the packed bundle holds +// the toolchain's packages only when the artifact keeps one external, never typescript or esbuild. +func TestThePackagesAreCopiedBesideTheTreeAndIntoTheBundleOnlyForExternals(t *testing.T) { + m, _, work := builtMachine(t, ``) + out, err := m.build(context.Background(), buildArgs()) + if err != nil { + t.Fatal(err) + } + nm := filepath.Join(work, "w1", "stage-tools", "node_modules") + for _, p := range []string{"@novox/mesh-sdk/dist/stdio.js", "@types/node/index.d.ts", "nats/lib/nats.js"} { + if _, err := os.Stat(filepath.Join(nm, p)); err != nil { + t.Errorf("the workspace's node_modules lacks %s", p) + } + } + for _, p := range []string{"typescript", "esbuild", "@esbuild"} { + if _, err := os.Stat(filepath.Join(nm, p)); err == nil { + t.Errorf("the toolchain's own %s was copied", p) + } + } + for f := range readArchive(t, out.(BuildResult).Archive) { + if strings.HasPrefix(f, "node_modules") { + t.Errorf("a bundle without externals carries %s", f) + } + } + m, _, _ = builtMachine(t, `,"external":["nats"]`) + out, _ = m.build(context.Background(), buildArgs()) + files := out.(BuildResult).Files + if !slices.Contains(files, "node_modules/nats/lib/nats.js") || !slices.Contains(files, "node_modules/@novox/mesh-sdk/dist/stdio.js") { + t.Errorf("files = %v, want the runtime packages beside the bundle", files) + } + for _, f := range files { + if strings.Contains(f, "typescript") || strings.Contains(f, "esbuild") { + t.Errorf("the bundle carries %s", f) + } + } +} + +func TestAFailedCompilerFailsTheStepNamingIt(t *testing.T) { + m, f, _ := builtMachine(t, ``) + f.answer["tsc"] = forgekey.Ran{Status: 2, Stdout: tscTypeError} + if _, err := m.build(context.Background(), buildArgs()); err == nil || !strings.Contains(err.Error(), "tsc for tools") || !strings.Contains(err.Error(), "tools/bad.ts") { + t.Errorf("err = %v", err) + } +} + +func TestOnlyATypeScriptBundleOfTheModuleIsBuilt(t *testing.T) { + m, _, _ := builtMachine(t, ``) + for _, a := range []string{"nope", ""} { + args := buildArgs() + args["artifact"] = a + if _, err := m.build(context.Background(), args); err == nil { + t.Errorf("artifact %q was built", a) + } + } +} + +// 1490 (review 7): the build agent runs a module mounted at /app/modules/module with the toolchain's packages at +// /app/node_modules (mesh-controller builder.go `within`), so esbuild writes the path comments of what it inlines +// relative to the module's directory as `../../node_modules/...`. The step builds at that depth: in a staging +// directory whose module directory has the packages two levels above it, whatever the module's place in the +// repository, so the bundle's bytes (and its digest) equal the build agent's. +func TestTheBuildRunsWithThePackagesTwoLevelsAboveTheModule(t *testing.T) { + for _, module := range []string{"modules/m", "."} { + m, f, work := newMachine(t) + files := map[string]string{"module.json": strings.Replace(bundleManifest, "%s", "", 1), "tools/index.ts": "export {}"} + if module != "." { + files = map[string]string{"modules/m/module.json": strings.Replace(bundleManifest, "%s", "", 1), "modules/m/tools/index.ts": "export {}"} + } + workspaceWith(t, work, files) + f.effect = esbuildEffect(t) + args := buildArgs() + args["module"] = module + if _, err := m.build(context.Background(), args); err != nil { + t.Fatalf("%s: %v", module, err) + } + for _, c := range f.calls { + rel, err := filepath.Rel(c.dir, filepath.Join(c.dir, "..", "..", "node_modules")) + if err != nil || rel != "../../node_modules" { + t.Fatalf("%s: relative path %q (%v)", module, rel, err) + } + if _, err := os.Stat(filepath.Join(c.dir, "..", "..", "node_modules", "@novox", "mesh-sdk", "dist", "stdio.js")); err != nil { + t.Errorf("%s: %s ran in %s, which has no packages at ../../node_modules: %v", module, filepath.Base(c.name), c.dir, err) + } + if filepath.Base(c.dir) != "module" || filepath.Base(filepath.Dir(c.dir)) != "modules" { + t.Errorf("%s: ran in %s, want .../modules/module", module, c.dir) + } + } + if !strings.HasPrefix(f.calls[0].dir, filepath.Join(work, "w1")) { + t.Errorf("%s: staged outside the workspace: %s", module, f.calls[0].dir) + } + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check.go b/modules/mesh-build-node/cmd/mesh-build-node/check.go new file mode 100644 index 00000000..672a3539 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/check.go @@ -0,0 +1,263 @@ +package main + +// The test step (novox/hq ADR 0445 §4; issues 1491 and 1292): the two levels of a TypeScript package's tests, run +// against a fetched workspace with the toolchain's tsc. The unit level is the package's type-check and its +// package.json `test` script; the integration level is its `test:integration` script, with the type-check as its +// setup. Each is timed and judged against its bound; a level over its bound says so in the result and does not fail +// (the measure of the catalogue's TypeScript is a later release's, R4f). A package with neither a script nor .ts files +// has no tests, and the result says so. +// +// **The bounds are the Go module's** (5 s for the unit level; 30 s of setup and 60 s of running for the integration +// level), because ADR 0445 names no figures for TypeScript. They are constants here, said in every result. + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// The bounds of a level (ADR 0445 §4). +const ( + UnitBound = 5 * time.Second + IntegrationSetupBound = 30 * time.Second + IntegrationRunBound = 60 * time.Second +) + +// TestResult is what a test step passes with, with the field names of mesh-build-go's (issue 1290). The step ends +// passed whenever the level ran, whatever its verdict, and Passed is the verdict; it ends failed only when the tests +// could not be run (no toolchain, a command that cannot start, a workspace that was never fetched). +type TestResult struct { + Workspace string `json:"workspace"` + Module string `json:"module"` + Level string `json:"level"` + Passed bool `json:"passed"` + // Why says, in words, why the level failed or went over its bound, or that it has no tests. + Why string `json:"why,omitempty"` + NoTests bool `json:"no_tests,omitempty"` + // Ran names what ran: "type-check", "test" or "test:integration". + Ran []string `json:"ran,omitempty"` + // Elapsed is the level's running time in seconds (at the unit level the type-check and the test script together); + // Setup is the integration level's setup, the type-check, in seconds. Bound and SetupBound are the bounds the + // level is judged by. + Elapsed float64 `json:"elapsed"` + Setup float64 `json:"setup,omitempty"` + Bound float64 `json:"bound"` + SetupBound float64 `json:"setup_bound,omitempty"` + // OverBound says a bound was exceeded. Unlike mesh-build-go's, an over-bound level still passes: the measure of + // the catalogue's TypeScript against the bounds is a later release's (R4f), not this one's gate. + OverBound bool `json:"over_bound"` + Failed []Failure `json:"failed,omitempty"` +} + +// Failure is a check or script that failed; Output is the first lines it wrote. +type Failure struct { + Package string `json:"package"` + Test string `json:"test,omitempty"` + Output string `json:"output,omitempty"` +} + +func (m *Module) testTool() StepTool { + return StepTool{ + Name: "build_node_test", + Description: "Run a TypeScript package's tests in a fetched workspace: the unit level is the type-check (the toolchain's " + + "tsc) and the package's test script, the integration level its test:integration script. Each level is timed " + + "and judged against its bound (unit 5 s; integration 30 s setup and 60 s running); a level over its bound " + + "says so and does not fail. A package with neither script nor .ts files has no tests. Answers a step id at " + + "once; the step ends on its key in the steps state.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id a fetch step answered"}, + "module": map[string]any{"type": "string", "description": "the package's directory in the workspace, '.' when it is the root"}, + "level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}}, + }, + "required": []string{"workspace", "level"}, + }, + Long: true, + Run: m.test, + } +} + +func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { + workspace, _ := args["workspace"].(string) + module, _ := args["module"].(string) + level, _ := args["level"].(string) + if level != "unit" && level != "integration" { + return nil, fmt.Errorf("level %q is neither unit nor integration", level) + } + root, dir, err := m.moduleDirOf(workspace, module) + if err != nil { + return nil, err + } + scripts, err := packageScripts(dir) + if err != nil { + return nil, err + } + sources, err := typeScriptFiles(dir) + if err != nil { + return nil, err + } + res := &TestResult{Workspace: workspace, Module: module, Level: level} + script := "test" + if level == "integration" { + script = "test:integration" + } + hasScript := scripts[script] != "" + if level == "integration" && !hasScript { + res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test:integration script" + return res, nil + } + if !hasScript && len(sources) == 0 { + res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test script and no .ts files" + return res, nil + } + if err := m.provide(root); err != nil { + return nil, fmt.Errorf("providing the toolchain's packages to the workspace: %w", err) + } + where := module + if where == "" { + where = "." + } + var check, run time.Duration + var failed []Failure + // a command that could not start is not a verdict: the step fails. One that ran and said no is. + ran := func(name string, r forgekey.Ran) error { + if r.Err != "" { + return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr)) + } + if r.Status != 0 { + failed = append(failed, Failure{Package: where, Test: name, Output: lines(r.Stdout + "\n" + r.Stderr)}) + } + return nil + } + if len(sources) > 0 { + start := m.now() + r := m.runIn(ctx, dir, nil, "node", m.typeCheck(dir, sources)...) + check = m.now().Sub(start) + if err := ran("type-check", r); err != nil { + return nil, err + } + res.Ran = append(res.Ran, "type-check") + } + if hasScript && len(failed) == 0 { + start := m.now() + env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm")} + r := m.runIn(ctx, dir, env, "npm", scriptArgs(script)...) + run = m.now().Sub(start) + if err := ran(script, r); err != nil { + return nil, err + } + res.Ran = append(res.Ran, script) + } + res.Failed = failed + var why []string + for _, f := range failed { + why = append(why, fmt.Sprintf("%s failed in %s: %s", f.Test, f.Package, firstLine(f.Output))) + } + if level == "unit" { + res.Elapsed, res.Bound = (check + run).Seconds(), UnitBound.Seconds() + res.OverBound = check+run > UnitBound + if res.OverBound { + why = append(why, fmt.Sprintf("the unit level took %.2f s, over its bound of %s", res.Elapsed, UnitBound)) + } + } else { + res.Setup, res.Elapsed = check.Seconds(), run.Seconds() + res.SetupBound, res.Bound = IntegrationSetupBound.Seconds(), IntegrationRunBound.Seconds() + res.OverBound = check > IntegrationSetupBound || run > IntegrationRunBound + if res.OverBound { + why = append(why, fmt.Sprintf("the integration level took %.2f s of setup and %.2f s of running, over its bounds of %s and %s", + res.Setup, res.Elapsed, IntegrationSetupBound, IntegrationRunBound)) + } + } + res.Passed = len(failed) == 0 + res.Why = strings.Join(why, "; ") + return res, nil +} + +// lines is the first lines of a command's output. +func lines(out string) string { + var keep []string + for _, l := range strings.Split(out, "\n") { + if l = strings.TrimSpace(l); l != "" { + keep = append(keep, l) + if len(keep) == 15 { + break + } + } + } + return strings.Join(keep, "\n") +} + +func firstLine(s string) string { + if i := strings.IndexByte(strings.TrimSpace(s), '\n'); i >= 0 { + return strings.TrimSpace(s)[:i] + } + return strings.TrimSpace(s) +} + +// scriptArgs is npm's line for a script: `npm test` for test, `npm run ` for another. +func scriptArgs(script string) []string { + if script == "test" { + return []string{"test"} + } + return []string{"run", script} +} + +// typeCheck is tsc's line for the type-check: the package's tsconfig.json when it has one, else the build agent's +// flags (toolchain.go:143-147) with --noEmit over every .ts file of the package. +func (m *Module) typeCheck(dir string, sources []string) []string { + if _, err := os.Stat(filepath.Join(dir, "tsconfig.json")); err == nil { + return []string{m.tsc(), "--noEmit", "-p", "."} + } + return append([]string{m.tsc(), "--noEmit", "--module", "NodeNext", "--moduleResolution", "NodeNext", + "--target", "ES2022", "--rootDir", "."}, sources...) +} + +// packageScripts are the scripts of the package.json in dir (none without one). +func packageScripts(dir string) (map[string]string, error) { + raw, err := os.ReadFile(filepath.Join(dir, "package.json")) + if os.IsNotExist(err) { + return map[string]string{}, nil + } + if err != nil { + return nil, err + } + var p struct { + Scripts map[string]string `json:"scripts"` + } + if err := json.Unmarshal(raw, &p); err != nil { + return nil, fmt.Errorf("the package.json is not JSON: %w", err) + } + return p.Scripts, nil +} + +// typeScriptFiles are the .ts files of the package (declaration files excepted), relative and sorted, without +// node_modules, hidden directories and a build's output. +func typeScriptFiles(dir string) ([]string, error) { + var out []string + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + name := d.Name() + if d.IsDir() { + if path != dir && (name == "node_modules" || strings.HasPrefix(name, ".")) { + return filepath.SkipDir + } + return nil + } + if strings.HasSuffix(name, ".ts") && !strings.HasSuffix(name, ".d.ts") { + rel, _ := filepath.Rel(dir, path) + out = append(out, filepath.ToSlash(rel)) + } + return nil + }) + return out, err +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go new file mode 100644 index 00000000..dc3e79fa --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go @@ -0,0 +1,199 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// tscTypeError is what tsc 5.9.3 printed for `export const n: number = "text";` in tools/bad.ts, run on this machine +// on 2026-10-20 with the flags of the type-check below (status 2). +const tscTypeError = "tools/bad.ts(1,14): error TS2322: Type 'string' is not assignable to type 'number'.\n" + +func testArgs(level string) map[string]any { + return map[string]any{"workspace": "w1", "module": ".", "level": level} +} + +// 1491/AC1: a package with a type error fails the unit level, naming the file. +func TestATypeErrorFailsTheUnitLevelNamingTheFile(t *testing.T) { + m, f, work := newMachine(t) + workspaceWith(t, work, map[string]string{"tools/bad.ts": `export const n: number = "text";`, "package.json": `{"name":"x"}`}) + f.answer["tsc"] = forgekey.Ran{Status: 2, Stdout: tscTypeError} + out, err := m.test(context.Background(), testArgs("unit")) + if err != nil { + t.Fatalf("a type error ended the step %v, want it finished with the verdict in its result", err) + } + r := out.(*TestResult) + if r.Passed || len(r.Failed) != 1 || !strings.Contains(r.Why, "tools/bad.ts") || !strings.Contains(r.Failed[0].Output, "TS2322") { + t.Fatalf("result = %+v, want passed false, naming tools/bad.ts", r) + } + if c := f.calls[0]; !strings.Contains(joined(c), "--noEmit") || !strings.Contains(joined(c), "typescript/bin/tsc") || + !strings.Contains(joined(c), "--moduleResolution NodeNext --target ES2022") || !strings.HasSuffix(joined(c), "tools/bad.ts") { + t.Errorf("the type-check ran %s", joined(c)) + } +} + +// 1491/AC2: a package with a passing test script passes the unit level, which reports its time; over its bound it +// says so and still passes. +func TestAPassingTestScriptPassesTheUnitLevelAndReportsItsTime(t *testing.T) { + m, f, work := newMachine(t) + workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "package.json": `{"scripts":{"test":"node --test"}}`}) + f.cost["tsc"], f.cost["npm"] = 400*time.Millisecond, 600*time.Millisecond + out, err := m.test(context.Background(), testArgs("unit")) + if err != nil { + t.Fatal(err) + } + r := out.(*TestResult) + if !r.Passed || r.NoTests || r.Elapsed != 1 || r.OverBound || r.Bound != 5 || strings.Join(r.Ran, ",") != "type-check,test" { + t.Errorf("result = %+v, want the type-check and the test script in 1 s within the bound", r) + } + last := f.calls[len(f.calls)-1] + if last.name != "npm" || strings.Join(last.args, " ") != "test" { + t.Errorf("ran %s, want npm test", joined(last)) + } + f.cost["npm"] = 6 * time.Second + out, err = m.test(context.Background(), testArgs("unit")) + if r := out.(*TestResult); err != nil || !r.OverBound { + t.Errorf("a slow unit level = %+v, %v, want passed and over_bound", r, err) + } +} + +// 1491/AC3: a package with neither script nor .ts files has no tests, and nothing runs. +func TestAPackageWithNeitherScriptNorTypeScriptHasNoTests(t *testing.T) { + m, f, work := newMachine(t) + workspaceWith(t, work, map[string]string{"package.json": `{"name":"x"}`, "node_modules/p/x.ts": "ignored", "types.d.ts": "declare const a: 1;"}) + for _, level := range []string{"unit", "integration"} { + out, err := m.test(context.Background(), testArgs(level)) + r := out.(*TestResult) + if err != nil || !r.Passed || !r.NoTests || !strings.Contains(r.Why, "no tests") { + t.Errorf("%s = %+v, %v, want the result to say no tests", level, r, err) + } + } + if len(f.calls) != 0 { + t.Errorf("%d commands ran for a package with no tests", len(f.calls)) + } +} + +// 1491/AC4: with test:integration present the integration level runs it, and reports the setup (the type-check) and +// the running time apart. +func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testing.T) { + m, f, work := newMachine(t) + workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "tsconfig.json": `{}`, + "package.json": `{"scripts":{"test":"x","test:integration":"node --test int"}}`}) + f.cost["tsc"], f.cost["npm"] = 2*time.Second, 40*time.Second + out, err := m.test(context.Background(), testArgs("integration")) + if err != nil { + t.Fatal(err) + } + r := out.(*TestResult) + if r.Setup != 2 || r.Elapsed != 40 || r.SetupBound != 30 || r.Bound != 60 || !r.Passed || r.OverBound || r.Level != "integration" { + t.Errorf("result = %+v, want setup 2 s and running 40 s apart", r) + } + last := f.calls[len(f.calls)-1] + if strings.Join(last.args, " ") != "run test:integration" { + t.Errorf("ran %s, want npm run test:integration", joined(last)) + } + if tsc := f.calls[0]; strings.Join(tsc.args[1:], " ") != "--noEmit -p ." { + t.Errorf("a package with a tsconfig was checked with %s", joined(tsc)) + } + f.cost["npm"] = 61 * time.Second + out, _ = m.test(context.Background(), testArgs("integration")) + if !out.(*TestResult).OverBound { + t.Errorf("61 s of running is within the bound") + } +} + +// The commands of a step get an environment built from scratch: no word of this process's. +func TestTheStepsCommandsGetAnEnvironmentOfTheirOwn(t *testing.T) { + t.Setenv("MESH_SECRET_TOKEN", "hunter2") + m, f, work := newMachine(t) + workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "package.json": `{"scripts":{"test":"x"}}`}) + if _, err := m.test(context.Background(), testArgs("unit")); err != nil { + t.Fatal(err) + } + for _, c := range f.calls { + for _, e := range c.env { + if strings.Contains(e, "hunter2") || strings.HasPrefix(e, "MESH_") { + t.Errorf("%s was given %s", joined(c), e) + } + } + if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") { + t.Errorf("%s env = %v, want PATH, HOME and the locale", c.name, c.env) + } + } +} + +// 1491: the step finishes with the verdict when the tests fail, and fails only when they could not be run: a failing +// test script is a verdict, a program that cannot start is not. +func TestAFailingScriptIsAVerdictAndACommandThatCannotStartIsAFailedStep(t *testing.T) { + m, f, work := newMachine(t) + workspaceWith(t, work, map[string]string{"package.json": `{"scripts":{"test":"x"}}`}) + f.answer["npm"] = forgekey.Ran{Status: 1, Stdout: "not ok 1 - adds\n"} + out, err := m.test(context.Background(), testArgs("unit")) + r, _ := out.(*TestResult) + if err != nil || r == nil || r.Passed || len(r.Failed) != 1 || r.Failed[0].Test != "test" || !strings.Contains(r.Failed[0].Output, "not ok 1") { + t.Errorf("a failing script = %+v, %v, want passed false and its output", r, err) + } + f.answer["npm"] = forgekey.Ran{Status: 127, Err: "ENOENT"} + if out, err := m.test(context.Background(), testArgs("unit")); err == nil { + t.Errorf("npm not installed = %v, want the step to fail", out) + } + if _, err := m.test(context.Background(), map[string]any{"workspace": "nope", "level": "unit"}); err == nil { + t.Errorf("a workspace never fetched did not fail the step") + } +} + +func TestAWorkspaceOutsideTheWorkDirectoryIsRefused(t *testing.T) { + m, _, _ := newMachine(t) + for _, a := range []map[string]any{ + {"workspace": "../etc", "level": "unit"}, + {"workspace": "w1", "module": "../x", "level": "unit"}, + {"workspace": "w1", "module": "/etc", "level": "unit"}, + {"workspace": "w1", "level": "smoke"}, + } { + if _, err := m.test(context.Background(), a); err == nil { + t.Errorf("%v was accepted", a) + } + } +} + +// 1491 (review 6): --no-bin-links leaves the toolchain no bin directory, so `npm run build` found no tsc or esbuild +// and exited 127, which read as a failed test. The workspace now has a bin directory of small shell scripts (files, +// not links) that call the toolchain's tsc and esbuild, first on the PATH of the test script. +func TestTheTestScriptFindsTscAndEsbuildOnItsPath(t *testing.T) { + m, f, work := newMachine(t) + workspaceWith(t, work, map[string]string{"package.json": `{"scripts":{"test":"tsc && esbuild"}}`}) + if _, err := m.test(context.Background(), testArgs("unit")); err != nil { + t.Fatal(err) + } + bin := filepath.Join(work, "w1", "bin") + nm := filepath.Join(m.toolchainDir(), "node_modules") + for name, want := range map[string]string{ + "tsc": "exec node '" + nm + "/typescript/bin/tsc' \"$@\"", + "esbuild": "exec '" + nm + "/esbuild/bin/esbuild' \"$@\"", + } { + info, err := os.Lstat(filepath.Join(bin, name)) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + t.Fatalf("bin/%s: %v %v, want an executable file", name, info, err) + } + raw, _ := os.ReadFile(filepath.Join(bin, name)) + if !strings.HasPrefix(string(raw), "#!/bin/sh\n") || !strings.Contains(string(raw), want) { + t.Errorf("bin/%s = %q, want a script calling the toolchain's program", name, raw) + } + } + npm := f.calls[len(f.calls)-1] + path := "" + for _, e := range npm.env { + if strings.HasPrefix(e, "PATH=") { + path = strings.TrimPrefix(e, "PATH=") // the last PATH wins + } + } + if npm.name != "npm" || !strings.HasPrefix(path, bin+":") { + t.Errorf("npm ran with PATH %q, want %s first", path, bin) + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/emit_test.go b/modules/mesh-build-node/cmd/mesh-build-node/emit_test.go new file mode 100644 index 00000000..5d98c239 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/emit_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +// Issues 1483 and 1488 (for the indexer, issue 1444): the guard emits toolchain.reported after each write of the +// toolchains key, with the node and the language. +func TestTheGuardEmitsToolchainReportedAfterEachWrite(t *testing.T) { + m := moduleWith(newStates(), "v26.10.0") + type event struct { + typ string + body map[string]string + } + var got []event + m.Emit = func(typ string, body any) error { + raw, _ := json.Marshal(body) + var b map[string]string + json.Unmarshal(raw, &b) + got = append(got, event{typ, b}) + return nil + } + if err := m.Check(context.Background()); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].typ != "toolchain.reported" || got[0].body["node"] != "ace" || got[0].body["language"] != "typescript" || len(got[0].body) != 2 { + t.Errorf("events = %+v, want one toolchain.reported for ace, typescript", got) + } + m.Check(context.Background()) + if len(got) != 2 { + t.Errorf("a second guard pass emitted %d events in all, want 2", len(got)) + } +} + +// The event is carried by the module's stream, not named in emits (ADR 0363: one place); the validator checks that. +func TestTheManifestDeclaresTheToolchainsStream(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Streams []struct { + Name string `json:"name"` + Events []string `json:"events"` + } `json:"streams"` + } + if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) != 1 || m.Streams[0].Name != "toolchains-node" || !slices.Equal(m.Streams[0].Events, []string{"toolchain.**"}) { + t.Errorf("streams = %+v (%v)", m.Streams, err) + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go new file mode 100644 index 00000000..687dc6b5 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go @@ -0,0 +1,298 @@ +package main + +// The fetch step (novox/hq ADR 0405 §5, ADR 0466 §4 and §5, ADR 0469 §2 and §5; issue 1489): a commit of a repository +// of the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and for each +// TypeScript module in it the packages its package.json names beyond the SDK, installed from its committed lock. +// +// **Only what the module itself depends on is installed.** The SDK and nats come with the toolchain, so a bundle that +// names only @novox/mesh-sdk (and build-time types and typescript) resolves nothing. One that names more must carry a +// package-lock.json, or the step fails naming its directory. The install is the build agent's (mesh-controller +// internal/builder/dependencies.go:49-95): in a scratch copy of package.json and the lock with the SDK, devDependencies +// and scripts taken out, `npm ci --omit=dev --omit=peer --ignore-scripts`, the install's copies of the SDK removed, and +// the result put at the module's node_modules. This module adds --no-bin-links, so the workspace holds no symlink. + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1). +const ForgeAlias = "mesh-forge" + +// sdkPackage is the package the toolchain carries and a module's own dependencies never supply. +const sdkPackage = "@novox/mesh-sdk" + +// WorkspaceAge is how long a workspace is kept. +const WorkspaceAge = 24 * time.Hour + +var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) + +// FetchResult is what a passed fetch step says. +type FetchResult struct { + Workspace string `json:"workspace"` + Commit string `json:"commit"` + // Modules are the TypeScript modules' directories found, and Installed those whose own packages were installed. + Modules []string `json:"modules"` + Installed []string `json:"installed"` + // Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK + // with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459). + Refused []Refusal `json:"refused"` +} + +// Refusal is a module a fetch did not install the packages of. +type Refusal struct { + Module string `json:"module"` + Reason string `json:"reason"` +} + +func (m *Module) fetchTool() StepTool { + return StepTool{ + Name: "build_node_fetch", + Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " + + "workspace of its own, and install each TypeScript module's own packages from its lock (a module naming only the " + + "SDK installs nothing). Answers a step id at once; the step ends on its key in the steps state with the workspace's id.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "/ on the forge"}, + "commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"}, + "ref": map[string]any{"type": "string", "description": "refs/pull//head when only a pull request's head carries the commit"}, + }, + "required": []string{"repository", "commit"}, + }, + Long: true, + Run: m.fetch, + } +} + +func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { + repository, _ := args["repository"].(string) + commit, _ := args["commit"].(string) + ref, _ := args["ref"].(string) + if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") { + return nil, fmt.Errorf("%q is not a repository, /", repository) + } + m.CleanWorkspaces(m.now()) + id := buildstep.NewID() + root := filepath.Join(m.Work, id) + tree := filepath.Join(root, "tree") + if err := os.MkdirAll(tree, 0o700); err != nil { + return nil, err + } + // git and ssh need PATH and the account's HOME, which a clean runner does not supply. + gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran { + return m.Run(ctx, append(m.baseEnv(), env...), name, a...) + } + remote := "ssh://" + ForgeAlias + "/" + repository + ".git" + if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil { + return nil, err + } + dirs, err := m.typeScriptModules(ctx, tree) + if err != nil { + return nil, err + } + res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}} + for _, d := range dirs { + res.Modules = append(res.Modules, d) + did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d) + if err != nil { + return nil, err + } + if refused != "" { + res.Refused = append(res.Refused, Refusal{Module: d, Reason: refused}) + } + if did { + res.Installed = append(res.Installed, d) + } + } + return res, nil +} + +// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to +// it, in order. +func (m *Module) typeScriptModules(ctx context.Context, tree string) ([]string, error) { + r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json") + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr)) + } + var dirs []string + for _, l := range strings.Split(r.Stdout, "\n") { + if l = strings.TrimSpace(l); l == "" { + continue + } + raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l))) + if err != nil { + return nil, err + } + var mj struct { + Build struct { + Artifacts []artifact `json:"artifacts"` + } `json:"build"` + } + if json.Unmarshal(raw, &mj) != nil { + continue // a module.json the build step will refuse by name, not a fetch's business + } + for _, a := range mj.Build.Artifacts { + if a.Language == "typescript" { + dirs = append(dirs, filepath.ToSlash(filepath.Dir(l))) + break + } + } + } + sort.Strings(dirs) + return dirs, nil +} + +// ownDependencies are the package.json's dependencies beyond the SDK and beyond what is build-time: the types and +// typescript, which the toolchain carries (dependencies.go:49-70). Sorted; none for a module without a package.json. +func ownDependencies(dir string) (names []string, pkg map[string]any, err error) { + raw, err := os.ReadFile(filepath.Join(dir, "package.json")) + if os.IsNotExist(err) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + if err := json.Unmarshal(raw, &pkg); err != nil { + return nil, nil, fmt.Errorf("the module's package.json is not JSON: %w", err) + } + deps, _ := pkg["dependencies"].(map[string]any) + for name := range deps { + if name != sdkPackage && name != "typescript" && !strings.HasPrefix(name, "@types/") { + names = append(names, name) + } + } + sort.Strings(names) + return names, pkg, nil +} + +// lockProblem says why a module's own packages cannot be installed: it names packages beyond the SDK and has no +// package-lock.json. Empty when it names none or has its lock. +func lockProblem(dir, rel string) (string, error) { + names, _, err := ownDependencies(dir) + if err != nil || len(names) == 0 { + return "", err + } + if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil { + return fmt.Sprintf("%s names packages beyond the SDK and has no package-lock.json (%s): commit one", rel, strings.Join(names, ", ")), nil + } + return "", nil +} + +// installOwn installs a module's own packages into its directory, or does nothing for a module that has none. A module +// that names packages and has no lock is not installed and not an error: the reason is answered, the fetch passes and +// the build of that bundle fails naming it. +func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (installed bool, refused string, err error) { + names, pkg, err := ownDependencies(dir) + if err != nil || len(names) == 0 { + return false, "", err + } + if why, err := lockProblem(dir, rel); why != "" || err != nil { + return false, why, err + } + lock, err := os.ReadFile(filepath.Join(dir, "package-lock.json")) + if err != nil { + return false, "", err + } + if m.Toolchain == nil || m.Toolchain.Registry == nil { + return false, "", fmt.Errorf("the module has no package registry to install %s from", rel) + } + reg, err := m.Toolchain.Registry() + if err != nil { + return false, "", err + } + // A scratch copy, so the module's package.json and lock are never rewritten (dependencies.go installSteps). + scratch := filepath.Join(root, "npm-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel)) + if err := os.MkdirAll(scratch, 0o700); err != nil { + return false, "", err + } + for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies"} { + if d, ok := pkg[k].(map[string]any); ok { + delete(d, sdkPackage) + } + } + delete(pkg, "devDependencies") + delete(pkg, "scripts") + raw, _ := json.Marshal(pkg) + if err := os.WriteFile(filepath.Join(scratch, "package.json"), raw, 0o600); err != nil { + return false, "", err + } + if err := os.WriteFile(filepath.Join(scratch, "package-lock.json"), lock, 0o600); err != nil { + return false, "", err + } + // The registry is given through a config file of its own, never on the command line, and removed after. + npmrc := filepath.Join(scratch, ".npmrc-install") + if err := os.WriteFile(npmrc, []byte(reg.Npmrc()), 0o600); err != nil { + return false, "", err + } + env := append(m.baseEnv(), "npm_config_cache="+filepath.Join(m.Toolchain.Paths.Cache, "npm")) + r := m.Run(ctx, env, "npm", "ci", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links", + "--no-audit", "--no-fund", "--prefix", scratch, "--userconfig", npmrc) + os.Remove(npmrc) + if why := ranFailed(r); why != "" { + return false, "", fmt.Errorf("installing the packages of %s (%s): %s", rel, strings.Join(names, ", "), why) + } + nm := filepath.Join(scratch, "node_modules") + if err := removeSDKCopies(nm); err != nil { + return false, "", err + } + if err := os.RemoveAll(filepath.Join(dir, "node_modules")); err != nil { + return false, "", err + } + if err := os.Rename(nm, filepath.Join(dir, "node_modules")); err != nil { + return false, "", err + } + return true, "", os.RemoveAll(scratch) +} + +// removeSDKCopies removes every copy of the SDK the install pulled in: the toolchain's is the pin. +func removeSDKCopies(nm string) error { + var found []string + err := filepath.WalkDir(nm, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && strings.HasSuffix(filepath.ToSlash(p), "/node_modules/@novox/mesh-sdk") { + found = append(found, p) + return fs.SkipDir + } + return nil + }) + if err != nil { + return err + } + for _, p := range found { + if err := os.RemoveAll(p); err != nil { + return err + } + } + return nil +} + +// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory. +func (m *Module) CleanWorkspaces(now time.Time) { + entries, err := os.ReadDir(m.Work) + if err != nil { + return + } + for _, e := range entries { + info, err := e.Info() + if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge { + continue + } + if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil { + m.Say("the workspace %s could not be removed: %v", e.Name(), err) + } + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go new file mode 100644 index 00000000..07ebeb09 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go @@ -0,0 +1,303 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +const commitSHA = "0123456789abcdef0123456789abcdef01234567" + +var fetchArgs = map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA} + +// fetchMachine is a machine whose git checks out the given files into the tree it is asked to fetch into (the fake +// answers the commands of FetchCommit and `git ls-files`), and whose npm ci writes the packages the test names. +func fetchMachine(t *testing.T, files map[string]string, installs map[string]string) (*Module, *fakeMachine) { + t.Helper() + m, f, _ := newMachine(t) + f.answer["git rev-parse"] = forgekey.Ran{Stdout: commitSHA + "\n"} + var manifests []string + for p := range files { + if strings.HasSuffix(p, "module.json") { + manifests = append(manifests, p) + } + } + f.answer["git ls-files"] = forgekey.Ran{Stdout: strings.Join(manifests, "\n") + "\n"} + f.effect = func(c call) { + switch { + case c.name == "git" && slices_contains(c.args, "checkout"): + for i, a := range c.args { + if a == "-C" { + for p, body := range files { + write(t, filepath.Join(c.args[i+1], filepath.FromSlash(p)), body, 0o644) + } + } + } + case c.name == "npm": + for i, a := range c.args { + if a == "--prefix" { + for p, body := range installs { + write(t, filepath.Join(c.args[i+1], "node_modules", filepath.FromSlash(p)), body, 0o644) + } + } + } + } + } + return m, f +} + +func slices_contains(l []string, s string) bool { + for _, x := range l { + if x == s { + return true + } + } + return false +} + +func typeScriptModule(pkg string) map[string]string { + return map[string]string{ + "modules/m/module.json": `{"module":"m","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`, + "modules/m/package.json": pkg, + } +} + +func npmCalls(f *fakeMachine) []call { + var out []call + for _, c := range f.calls { + if c.name == "npm" { + out = append(out, c) + } + } + return out +} + +// 1489/AC1: a module whose package.json names only @novox/mesh-sdk and @types/node (and typescript) installs nothing +// and the step passes. +func TestAModuleNamingOnlyTheSdkAndTypesInstallsNothing(t *testing.T) { + m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"},"devDependencies":{"@types/node":"^22","typescript":"^5"}}`), nil) + out, err := m.fetch(context.Background(), fetchArgs) + if err != nil { + t.Fatal(err) + } + r := out.(FetchResult) + if len(npmCalls(f)) != 0 || len(r.Installed) != 0 || len(r.Modules) != 1 || r.Modules[0] != "modules/m" || r.Commit != commitSHA { + t.Errorf("result = %+v, npm calls %d, want the module found and nothing installed", r, len(npmCalls(f))) + } + if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "module.json")); err != nil { + t.Errorf("the commit is not checked out in the workspace: %v", err) + } +} + +// 1489/AC2: a module naming lodash with a lock gets npm ci --omit=dev --ignore-scripts (fake runner), its install put +// at its node_modules without the SDK and without any copy of it; without a lock the step fails naming the directory. +func TestAModuleNamingLodashWithALockIsInstalledWithoutTheSdkAndWithoutALockFails(t *testing.T) { + files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"},"devDependencies":{"x":"1"},"scripts":{"test":"y"}}`) + files["modules/m/package-lock.json"] = `{"lockfileVersion":3}` + m, f := fetchMachine(t, files, map[string]string{"lodash/index.js": "l", "@novox/mesh-sdk/index.js": "sdk", "a/node_modules/@novox/mesh-sdk/i.js": "nested"}) + out, err := m.fetch(context.Background(), fetchArgs) + if err != nil { + t.Fatal(err) + } + r := out.(FetchResult) + calls := npmCalls(f) + if len(calls) != 1 { + t.Fatalf("npm ran %d times, want once", len(calls)) + } + line := strings.Join(calls[0].args, " ") + if !strings.HasPrefix(line, "ci --omit=dev --omit=peer --ignore-scripts --no-bin-links") { + t.Errorf("ran npm %s", line) + } + nm := filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules") + if _, err := os.Stat(filepath.Join(nm, "lodash", "index.js")); err != nil { + t.Errorf("lodash is not at the module's node_modules: %v", err) + } + for _, gone := range []string{"@novox/mesh-sdk", "a/node_modules/@novox/mesh-sdk"} { + if _, err := os.Stat(filepath.Join(nm, gone)); err == nil { + t.Errorf("a copy of the SDK is left at %s", gone) + } + } + if got := strings.Join(r.Installed, ","); got != "modules/m" { + t.Errorf("installed = %q", got) + } + // the module's own package.json is untouched: the install worked on a scratch copy. + if raw, _ := os.ReadFile(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "package.json")); !strings.Contains(string(raw), "scripts") { + t.Errorf("the module's package.json was rewritten: %s", raw) + } + var scratch []string + for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) { + scratch = append(scratch, e) + } + if strings.Contains(strings.Join(scratch, ","), "npm-") { + t.Errorf("the scratch install is left in the workspace: %v", scratch) + } + +} + +// 1489 (the 1459 settlement): a module naming packages with no package-lock.json does not fail the fetch: it passes, +// installs nothing for that module, and its result lists the refused bundle with its reason; a build of that bundle +// fails naming it. +func TestAModuleWithoutALockIsRefusedInTheResultNotTheFetch(t *testing.T) { + files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"}}`) + files["modules/ok/module.json"] = `{"module":"ok","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}` + files["modules/ok/package.json"] = `{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}` + m, f := fetchMachine(t, files, nil) + out, err := m.fetch(context.Background(), fetchArgs) + if err != nil { + t.Fatalf("a module without a lock failed the fetch: %v", err) + } + r := out.(FetchResult) + if len(npmCalls(f)) != 0 || len(r.Installed) != 0 || len(r.Modules) != 2 { + t.Errorf("result = %+v, npm calls %d", r, len(npmCalls(f))) + } + if len(r.Refused) != 1 || r.Refused[0].Module != "modules/m" || + !strings.Contains(r.Refused[0].Reason, "names packages beyond the SDK and has no package-lock.json") || !strings.Contains(r.Refused[0].Reason, "lodash") { + t.Fatalf("refused = %+v, want modules/m with its reason", r.Refused) + } + // the build of that bundle fails naming it, without running a compiler. + f.calls = nil + _, err = m.build(context.Background(), map[string]any{"workspace": r.Workspace, "module": "modules/m", "artifact": "tools"}) + if err == nil || !strings.Contains(err.Error(), "modules/m") || !strings.Contains(err.Error(), "no package-lock.json") || len(f.calls) != 0 { + t.Errorf("build = %v after %d commands, want the refusal naming modules/m and nothing run", err, len(f.calls)) + } +} + +// 1489 (review 4): a bundle depending on another @novox package is installed, the mesh's registry given to npm ci +// through --userconfig, as the toolchain's install gives it. +func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *testing.T) { + files := typeScriptModule(`{"dependencies":{"@novox/other":"^1","lodash":"^4"}}`) + files["modules/m/package-lock.json"] = `{}` + m, f := fetchMachine(t, files, map[string]string{"@novox/other/index.js": "o", "lodash/index.js": "l"}) + m.Toolchain.Registry = func() (Registry, error) { + return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil + } + var npmrc string + inner := f.effect + f.effect = func(c call) { + inner(c) + for i, a := range c.args { + if c.name == "npm" && a == "--userconfig" { + raw, _ := os.ReadFile(c.args[i+1]) + npmrc = string(raw) + } + } + } + out, err := m.fetch(context.Background(), fetchArgs) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || !strings.Contains(npmrc, ":_auth=") { + t.Errorf("npm ci was given the config %q", npmrc) + } + if strings.Contains(strings.Join(npmCalls(f)[0].args, " "), "pw") { + t.Errorf("the credential is on npm's command line") + } + r := out.(FetchResult) + if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules", "@novox", "other", "index.js")); err != nil { + t.Errorf("@novox/other was not installed: %v", err) + } + for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) { + if strings.HasPrefix(e, "npm-") { + t.Errorf("the scratch install (and its credential file) is left: %s", e) + } + } +} + +func mustReadDir(t *testing.T, dir string) []string { + t.Helper() + es, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var out []string + for _, e := range es { + out = append(out, e.Name()) + } + return out +} + +// 1489/AC3: given a process environment holding a token, no command of the step sees it. The commands the step makes +// are given PATH, HOME and the locale and their own words; and the real clean runner, run on `env` here, prints only +// those. +func TestNoCommandOfTheStepSeesATokenOfTheProcess(t *testing.T) { + t.Setenv("MESH_BUS_CREDENTIAL", "hunter2") + t.Setenv("NPM_TOKEN", "hunter3") + files := typeScriptModule(`{"dependencies":{"lodash":"^4"}}`) + files["modules/m/package-lock.json"] = `{}` + m, f := fetchMachine(t, files, map[string]string{"lodash/index.js": "l"}) + if _, err := m.fetch(context.Background(), fetchArgs); err != nil { + t.Fatal(err) + } + if len(f.calls) < 6 { + t.Fatalf("only %d commands ran", len(f.calls)) + } + for _, c := range f.calls { + for _, e := range c.env { + if strings.Contains(e, "hunter") || strings.HasPrefix(e, "MESH_") || strings.HasPrefix(e, "NPM_TOKEN") { + t.Errorf("%s %v was given %s", c.name, c.args[:min(len(c.args), 3)], e) + } + } + if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") || c.env[2] != "LC_ALL=C" { + t.Errorf("%s was given %v, want PATH, HOME and the locale first", c.name, c.env) + } + } + // The runner main wires: what it hands a command is exactly what the step gave. + ran := forgekey.ExecRunnerClean(10*time.Second)(context.Background(), append(m.baseEnv(), "ONLY=this"), "env") + if ran.Status != 0 || strings.Contains(ran.Stdout, "hunter") || !strings.Contains(ran.Stdout, "ONLY=this") { + t.Errorf("the clean runner printed %q", ran.Stdout) + } +} + +func TestFetchRefusesWhatIsNotARepositoryBeforeAnythingRuns(t *testing.T) { + m, f := fetchMachine(t, nil, nil) + for _, repo := range []string{"", "x", "../etc/passwd", "a/b/c", "a/..", "o/r;rm"} { + if _, err := m.fetch(context.Background(), map[string]any{"repository": repo, "commit": commitSHA}); err == nil { + t.Errorf("%q was accepted", repo) + } + } + if len(f.calls) != 0 { + t.Errorf("%d commands ran", len(f.calls)) + } +} + +func TestFetchAsksGitForTheSshRemote(t *testing.T) { + m, f := fetchMachine(t, typeScriptModule(`{}`), nil) + if _, err := m.fetch(context.Background(), fetchArgs); err != nil { + t.Fatal(err) + } + var all []string + for _, c := range f.calls { + all = append(all, c.name+" "+strings.Join(c.args, " ")) + } + lines := strings.Join(all, "\n") + for _, want := range []string{"remote add origin ssh://mesh-forge/novox/mesh-catalog.git", "fetch --no-tags --depth 1 origin " + commitSHA, "checkout -q --detach " + commitSHA} { + if !strings.Contains(lines, want) { + t.Errorf("git was not asked %q:\n%s", want, lines) + } + } +} + +func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) { + m, _, work := newMachine(t) + old, fresh := filepath.Join(work, "aaaa"), filepath.Join(work, "bbbb") + for _, d := range []string{old, fresh, filepath.Join(work, "Not-A-Workspace")} { + os.MkdirAll(d, 0o755) + } + os.Chtimes(old, time.Now().Add(-25*time.Hour), time.Now().Add(-25*time.Hour)) + os.Chtimes(filepath.Join(work, "Not-A-Workspace"), time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour)) + m.CleanWorkspaces(time.Now()) + if _, err := os.Stat(old); err == nil { + t.Errorf("the old workspace is kept") + } + if _, err := os.Stat(fresh); err != nil { + t.Errorf("the fresh workspace was removed") + } + if _, err := os.Stat(filepath.Join(work, "Not-A-Workspace")); err != nil { + t.Errorf("a directory that is no workspace was removed") + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/forge_test.go b/modules/mesh-build-node/cmd/mesh-build-node/forge_test.go new file mode 100644 index 00000000..55d0a83f --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/forge_test.go @@ -0,0 +1,247 @@ +package main + +// The forge side of a build module (novox/hq ADR 0466 §2 and §4, issue 1484). This file is the same for every build +// module: mesh-build-node takes a copy and changes the three constants below. + +import ( + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +const ( + forgeModule = "mesh-build-node" + forgeHome = "/var/lib/mesh-build-node" + forgeModuleD = "../.." // the module's directory, where module.json and files/ are +) + +type forgeManifest struct { + Requires []any `json:"requires"` + State []any `json:"state"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` +} + +func readForgeManifest(t *testing.T) forgeManifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join(forgeModuleD, "module.json")) + if err != nil { + t.Fatal(err) + } + var m forgeManifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +func forgeTemplate(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(forgeModuleD, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +func forgeResource(t *testing.T, m forgeManifest, id string) map[string]any { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + if r["owner"] != forgeModule { + t.Errorf("%s is owned by %v, not the module's own account", id, r["owner"]) + } + if p, _ := r["path"].(string); !strings.HasPrefix(p, "${dir:") || strings.Contains(p, "account-home") { + t.Errorf("%s is at %q: a path of the account's own home (${dir:...}), never the operator's", id, p) + } + return r + } + } + t.Fatalf("no resource %q", id) + return nil +} + +var forgeSample = map[string]string{ + "known_host": "[novox.internal]:222 ssh-ed25519 AAAAC3Nza", + "host_key": "ssh-ed25519 AAAAC3Nza", + "ssh_host": "novox.internal", + "ssh_alias": "mesh-forge", + "ssh_port": "222", + "ssh_base": "ssh://git@novox.internal:222/", + "https_base": "https://git.novox.be/", +} + +// forgeRender fills ${bound:forge-account:} with the sample values, as the provisioner client does. +func forgeRender(t *testing.T, s string) string { + t.Helper() + for k, v := range forgeSample { + s = strings.ReplaceAll(s, "${bound:forge-account:"+k+"}", v) + } + if strings.Contains(s, "${") { + t.Errorf("unfilled placeholder in %q", s) + } + return s +} + +// 1484/AC1: given sample bound values, the ssh config has Host mesh-forge, IdentitiesOnly yes and the key under the +// module's own home; known_hosts carries the host key; git's config rewrites https_base to ssh://mesh-forge/; no +// ${ is left. +func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) { + m := readForgeManifest(t) + + cfg := forgeResource(t, m, "ssh-config") + want := "Host mesh-forge\n HostName novox.internal\n Port 222\n HostKeyAlias mesh-forge\n User git\n IdentityFile ~/.ssh/mesh_ed25519\n IdentitiesOnly yes\n" + if got := forgeRender(t, forgeTemplate(t, cfg)); got != want { + t.Errorf("ssh config = %q, want %q", got, want) + } + if cfg["path"] != "${dir:ssh-dir}/config" || cfg["mode"] != "0600" || cfg["into"] != nil { + t.Errorf("ssh config resource = %v, want the whole file ${dir:ssh-dir}/config at mode 0600", cfg) + } + + kh := forgeResource(t, m, "known-hosts") + if got := strings.TrimSpace(forgeRender(t, forgeTemplate(t, kh))); got != "mesh-forge ssh-ed25519 AAAAC3Nza" { + t.Errorf("known_hosts = %q", got) + } + + g := forgeResource(t, m, "git-config") + wantGit := "[url \"ssh://mesh-forge/\"]\n\tinsteadOf = https://git.novox.be/\n" + if got := forgeRender(t, forgeTemplate(t, g)); got != wantGit { + t.Errorf("git config = %q, want %q", got, wantGit) + } + + dir := forgeResource(t, m, "ssh-dir") + if dir["mode"] != "0700" || dir["path"] != "${dir:state}/.ssh" { + t.Errorf("ssh-dir = %v, want ${dir:state}/.ssh at mode 0700", dir) + } +} + +// 1484/AC2: the module requires forge-account, keeps the state keys, and every file resource takes its text from a +// template in the files/ archive (novox/hq ADR 0349 §2). +func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) { + m := readForgeManifest(t) + if !slices.Contains(m.Requires, any("forge-account")) { + t.Errorf("requires = %v, want forge-account among them", m.Requires) + } + keeps := false + for _, s := range m.State { + if s == "keys" { + keeps = true + } + } + if !keeps { + t.Errorf("state = %v lacks keys", m.State) + } + files := 0 + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline", r["id"]) + } + if r["type"] == "file" { + files++ + forgeTemplate(t, r) + } + } + if files == 0 { + t.Errorf("no file resource: the forge's ssh files are missing") + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" && a["name"] == "files" { + packs++ + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives named files, not one", packs) + } +} + +// forgeEd25519Line is a public key line with a well-formed ed25519 blob. +func forgeEd25519Line(comment string) string { + blob := []byte{} + for _, part := range [][]byte{[]byte("ssh-ed25519"), make([]byte, 32)} { + l := make([]byte, 4) + binary.BigEndian.PutUint32(l, uint32(len(part))) + blob = append(append(blob, l...), part...) + } + return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(blob) + " " + comment +} + +type forgeKeys struct{ puts map[string][]byte } + +func (f *forgeKeys) Put(key string, value any) (uint64, error) { + raw, _ := json.Marshal(value) + if f.puts == nil { + f.puts = map[string][]byte{} + } + f.puts[key] = raw + return uint64(len(f.puts)), nil +} + +// forgeKeygen stands in for ssh-keygen: it writes the pair the command names. +func forgeKeygen(_ context.Context, _ []string, name string, args ...string) forgekey.Ran { + var file, comment string + for i, a := range args { + switch a { + case "-f": + file = args[i+1] + case "-C": + comment = args[i+1] + } + } + if name != "ssh-keygen" || file == "" { + return forgekey.Ran{Status: 1, Stderr: "unexpected"} + } + os.WriteFile(file, []byte("PRIVATE\n"), 0o600) + os.WriteFile(file+".pub", []byte(forgeEd25519Line(comment)+"\n"), 0o644) + return forgekey.Ran{} +} + +// 1484/AC3: given the bundle started with an empty home, the keeper writes . with kind builder, and +// the key is in the home's .ssh. +func TestTheKeeperWritesTheModulesEntryWithKindBuilder(t *testing.T) { + home, st := t.TempDir(), &forgeKeys{} + if err := keeperFor(home, "ace", forgeModule, forgeKeygen, st).Once(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(home, ".ssh", "mesh_ed25519")); err != nil { + t.Errorf("no key in an empty home: %v", err) + } + raw, ok := st.puts["ace."+forgeModule] + if !ok || len(st.puts) != 1 { + t.Fatalf("entries = %v, want one at ace.%s", st.puts, forgeModule) + } + var e forgekey.KeyEntry + if err := json.Unmarshal(raw, &e); err != nil { + t.Fatal(err) + } + if e.Kind != "builder" || e.Node != "ace" || e.Account != forgeModule || !strings.HasPrefix(e.Public, "ssh-ed25519 ") { + t.Errorf("entry = %s", raw) + } + if strings.Contains(strings.ToUpper(string(raw)), "PRIVATE") { + t.Errorf("the entry carries private material: %s", raw) + } +} + +// The home the manifest places is the one the account is given, so the keeper's home and the templates' agree. +func TestTheAccountsHomeIsTheOneTheManifestNames(t *testing.T) { + m := readForgeManifest(t) + for _, r := range m.Resources { + if r["type"] == "user" && r["name"] == forgeModule && r["home"] != forgeHome { + t.Errorf("the account's home is %v, want %s", r["home"], forgeHome) + } + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/harness_test.go b/modules/mesh-build-node/cmd/mesh-build-node/harness_test.go new file mode 100644 index 00000000..7ff120c7 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/harness_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// A fake machine for the step tests: a toolchain directory laid out as `npm ci --no-bin-links` leaves it, a work +// directory, a clock that only the fake commands advance, and a runner that records every command and answers by +// what the test set. + +type call struct { + dir string // env -C + name string // the program after env -C + args []string // its arguments + env []string +} + +type fakeMachine struct { + mu sync.Mutex + calls []call + now time.Time + // cost is how long a program takes, by the base name of its path; answer is what it prints and its status; + // effect runs the program's effects on the disk. + cost map[string]time.Duration + answer map[string]forgekey.Ran + effect func(c call) +} + +func (f *fakeMachine) run(_ context.Context, env []string, name string, args ...string) forgekey.Ran { + f.mu.Lock() + defer f.mu.Unlock() + c := call{name: name, args: args, env: env} + key := filepath.Base(name) + if name == "env" && len(args) >= 3 && args[0] == "-C" { + c = call{dir: args[1], name: args[2], args: args[3:], env: env} + key = filepath.Base(c.name) + if key == "node" && len(c.args) > 0 { + key = filepath.Base(c.args[0]) // node + } + } else if name == "git" { + for _, a := range args { + if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files"}, a) { + key = "git " + a + break + } + } + } + f.calls = append(f.calls, c) + f.now = f.now.Add(f.cost[key]) + if f.effect != nil { + f.effect(c) + } + return f.answer[key] +} + +func (f *fakeMachine) clock() time.Time { f.mu.Lock(); defer f.mu.Unlock(); return f.now } + +// keys of the programs: tsc, esbuild, npm. + +func newMachine(t *testing.T) (*Module, *fakeMachine, string) { + t.Helper() + home, cache, work := t.TempDir(), t.TempDir(), t.TempDir() + f := &fakeMachine{now: time.Unix(1_000_000, 0), cost: map[string]time.Duration{}, answer: map[string]forgekey.Ran{}} + m := &Module{Node: "ace", Recipe: Recipe{Steps: "1", Node: "v26.10.0", Lock: "lock"}, + Toolchain: &Toolchain{Paths: Paths{Home: home, Cache: cache}, Registry: func() (Registry, error) { + return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil + }}, + Work: work, Run: f.run, Clock: f.clock, Ctx: context.Background()} + // The toolchain directory as npm ci leaves it, with no bin links. + nm := filepath.Join(m.toolchainDir(), "node_modules") + for _, p := range []string{"typescript/bin/tsc", "esbuild/bin/esbuild", "@esbuild/linux-x64/bin/esbuild", + "@novox/mesh-sdk/dist/stdio.js", "@types/node/index.d.ts", "nats/lib/nats.js"} { + write(t, filepath.Join(nm, p), "x", 0o644) + } + return m, f, work +} + +func write(t *testing.T, path, body string, mode os.FileMode) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatal(err) + } +} + +// workspaceWith makes workspace "w1" of the machine with the files given (path under tree -> body). +func workspaceWith(t *testing.T, work string, files map[string]string) string { + t.Helper() + for p, body := range files { + write(t, filepath.Join(work, "w1", "tree", filepath.FromSlash(p)), body, 0o644) + } + return "w1" +} + +func symlinksUnder(t *testing.T, root string) []string { + t.Helper() + var out []string + filepath.Walk(root, func(p string, info os.FileInfo, err error) error { + if err == nil && info.Mode()&os.ModeSymlink != 0 { + out = append(out, p) + } + return nil + }) + return out +} + +func joined(c call) string { return c.name + " " + strings.Join(c.args, " ") } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/keys.go b/modules/mesh-build-node/cmd/mesh-build-node/keys.go new file mode 100644 index 00000000..9c92386d --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/keys.go @@ -0,0 +1,42 @@ +package main + +// The module's own forge key (novox/hq ADR 0466 §2, issue 1484): mesh-sdk's key keeper, kind builder, for the +// module's account in its home. It makes ~/.ssh/mesh_ed25519 when it is missing and writes one entry to this +// module's `keys` state at ., which gitea's provider reads to register the key on the forge user +// that reads and pushes nothing (ADR 0466 §3). + +import ( + "context" + "fmt" + "os" + "os/user" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// keeperFor is this module's keeper: go/forgekey's, of kind builder, for account in home on node, whose entry lies +// at . of the keys state. +func keeperFor(home, node, account string, run forgekey.Runner, state forgekey.KeyPutter) *forgekey.KeyKeeper { + return &forgekey.KeyKeeper{Home: home, Node: node, Account: account, Kind: forgekey.KindBuilder, Run: run, State: state, Now: time.Now} +} + +// startKeeper runs the keeper when the runtime launched this process for a node: outside a launch — a build, a +// check — there is no state to put the entry in. +func startKeeper(ctx context.Context) { + node := strings.TrimSpace(os.Getenv("MESH_NODE")) + u, err := user.Current() + if node == "" || err != nil || u.HomeDir == "" { + return + } + k := keeperFor(u.HomeDir, node, u.Username, forgekey.ExecRunner, keysState{}) + k.Keep(ctx, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-node] "+f+"\n", a...) }) +} + +type keysState struct{} + +func (keysState) Put(key string, value any) (uint64, error) { + return stdio.State("keys").Put(key, value) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go new file mode 100644 index 00000000..0e148a4f --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -0,0 +1,62 @@ +// mesh-build-node: the Node and TypeScript build module (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470; design +// 47 R4c-3, issue 1488). A Go tools bundle the node's runtime launches as the module's own account (runs-as). It names +// the one Node version it builds with, installs the TypeScript toolchain its own lock names, writes the Node version +// it finds to .typescript of its `toolchains` state at start and before every step, and refuses every step on +// a node that has another: its health is then unhealthy, so the controller raises the node's condition. What it +// says, it says on stderr; stdout is the MCP channel. +package main + +import ( + "context" + "fmt" + "os" + "os/user" + "path/filepath" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// CacheDir is the cache directory, outside the account's home (ADR 0466 §1; module.json resource `cache`). +const CacheDir = "/var/lib/mesh-build-node-cache" + +// WorkDir is the work directory, outside the home (module.json resource `work`). +const WorkDir = "/var/lib/mesh-build-node-work" + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-node] "+format+"\n", a...) } + +func main() { + ctx := context.Background() + home := "" + if u, err := user.Current(); err == nil { + home = u.HomeDir + } + paths := Paths{Home: home, Cache: CacheDir} + registry := func() (Registry, error) { + return ReadRegistry(filepath.Join(home, "package-registry.json"), filepath.Join(home, "package-registry.secret")) + } + m := &Module{ + Node: strings.TrimSpace(os.Getenv("MESH_NODE")), + Reload: func() (Recipe, error) { return ReadRecipe(paths.LockPath()) }, + Probe: ProbeNode, + Toolchain: &Toolchain{Paths: paths, Run: forgekey.ExecRunnerClean(0), Registry: registry}, + Toolchains: stdio.State(delivery.ToolchainsState).Put, + Emit: func(t string, b any) error { return stdio.Emit(t, b) }, + Steps: stdio.State(delivery.StepsState).Put, + Work: WorkDir, + Run: forgekey.ExecRunnerClean(0), + Ctx: ctx, + Say: logf, + } + // Outside a launch by the runtime — a build, a check — there is no node and no state to write. + if m.Node != "" { + go m.Start(ctx) + } + go startKeeper(ctx) + if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go new file mode 100644 index 00000000..8fb06a2d --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go @@ -0,0 +1,193 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The catalogue's root and this module's directory. +const ( + catalogueRoot = "../../../.." + moduleDir = "../.." +) + +func readManifest(t *testing.T) catalogue.Manifest { + t.Helper() + m, err := catalogue.ReadTreeManifest(catalogueRoot, moduleDir) + if err != nil { + t.Fatalf("mesh-sdk's validator refuses module.json: %v", err) + } + return m +} + +func resource(t *testing.T, m catalogue.Manifest, id string) map[string]any { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + return r + } + } + t.Fatalf("module.json declares no resource %q", id) + return nil +} + +// 1488/AC1: mesh-sdk's validator accepts the module (alone on a shelf too), and it runs as mesh-build-node with its +// home at mode 0700, requires user-namespaces and declares the four packages and carries toolchain/package-lock.json. +func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *testing.T) { + m := readManifest(t) + if problems := catalogue.CatalogueProblems(catalogue.Shelf{m.Module: m}); len(problems) > 0 { + t.Errorf("the catalogue's rules refuse it: %v", problems) + } + if m.Module != "mesh-build-node" || m.RunsAs != "mesh-build-node" { + t.Errorf("module %q runs as %q, want mesh-build-node as itself", m.Module, m.RunsAs) + } + account := resource(t, m, "account") + if account["type"] != "user" || account["name"] != "mesh-build-node" || account["shell"] != "/usr/bin/nologin" || account["home"] != "/var/lib/mesh-build-node" { + t.Errorf("account = %v", account) + } + home := resource(t, m, "state") + if home["type"] != "directory" || home["mode"] != "0700" || home["owner"] != "mesh-build-node" || home["place"] != "." { + t.Errorf("home = %v, want its directory at mode 0700 owned by the account", home) + } + if !slices.Contains(m.Capabilities, "user-namespaces") { + t.Errorf("capabilities = %v, want user-namespaces", m.Capabilities) + } + packages := []string{} + for _, r := range m.Resources { + if r["type"] == "package" { + packages = append(packages, r["package"].(string)) + } + } + slices.Sort(packages) + if want := []string{"git", "nodejs", "npm", "openssh"}; !slices.Equal(packages, want) { + t.Errorf("packages = %v, want %v", packages, want) + } +} + +// The work and cache directories are outside the home (ADR 0466 §1), owned by the account. +func TestWorkAndCacheDirectoriesLieOutsideTheHome(t *testing.T) { + m := readManifest(t) + for _, id := range []string{"work", "cache"} { + r := resource(t, m, id) + path, _ := r["path"].(string) + if path == "" || path == "/var/lib/mesh-build-node" || len(path) > len("/var/lib/mesh-build-node") && path[:len("/var/lib/mesh-build-node/")] == "/var/lib/mesh-build-node/" { + t.Errorf("%s is at %q, inside or at the home", id, path) + } + if r["owner"] != "mesh-build-node" { + t.Errorf("%s is owned by %v", id, r["owner"]) + } + } +} + +// The states are the three the module keeps, no seat is claimed (ADR 0469 §6), and the health is the module's own +// tool, which the manifest also lists among its tools. +func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) { + m := readManifest(t) + states := []string{} + for _, s := range m.State { + states = append(states, s.Name) + } + slices.Sort(states) + if want := []string{"keys", "steps", "toolchains"}; !slices.Equal(states, want) { + t.Errorf("state = %v, want %v", states, want) + } + if len(m.Claims) != 0 { + t.Errorf("claims = %v, want no seat", m.Claims) + } + if !slices.Contains(m.Tools, HealthTool) { + t.Errorf("tools = %v lack %s", m.Tools, HealthTool) + } + raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) { + t.Errorf("runtime-health does not name %s", HealthTool) + } +} + +// Every tool the manifest names is served, and every tool served is named. +func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) { + m := readManifest(t) + served := []string{} + for _, tool := range (&Module{}).Tools((&Module{}).StepTools()) { + served = append(served, tool.Name) + } + slices.Sort(served) + named := slices.Clone(m.Tools) + slices.Sort(named) + if !slices.Equal(served, named) { + t.Errorf("served %v, manifest names %v", served, named) + } +} + +// 1488/AC1: the module carries the committed lock beside its package.json, and the manifest unpacks them into the +// home's toolchain directory, where the program reads them. +func TestTheModuleCarriesItsToolchainLockAndTheManifestPlacesIt(t *testing.T) { + m := readManifest(t) + for _, name := range []string{"package.json", "package-lock.json"} { + if _, err := os.Stat(filepath.Join(moduleDir, "toolchain", name)); err != nil { + t.Errorf("the module carries no toolchain/%s: %v", name, err) + } + } + if !slices.Contains(m.Requires, "npm-package-registry") { + t.Errorf("requires = %v lack npm-package-registry, which the install resolves @novox from", m.Requires) + } +} + +// 1488/AC1: the lock names the versions of the recipe: typescript 5.9.3, esbuild 0.25.12, @novox/mesh-sdk 0.1.14 +// and nats, all by the lock's own hash (read from the file the module carries). +func TestTheLockNamesTheToolchainVersions(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "toolchain", "package-lock.json")) + if err != nil { + t.Fatal(err) + } + var lock struct { + Packages map[string]struct { + Version string `json:"version"` + } `json:"packages"` + } + if err := json.Unmarshal(raw, &lock); err != nil { + t.Fatal(err) + } + for pkg, want := range map[string]string{"typescript": "5.9.3", "esbuild": "0.25.12", "@novox/mesh-sdk": "0.1.14", "nats": "2.29.3", "@types/node": "22.20.1"} { + if got := lock.Packages["node_modules/"+pkg].Version; got != want { + t.Errorf("%s = %q in the lock, want %s", pkg, got, want) + } + } +} + +// 1488 (the catalogue's manifest check): a manifest names no host, the mesh being name-agnostic. The tree-read +// manifest carries each template's text as the resource's content, and the toolchain's lock names registries, so it is +// shipped as an archive artifact instead, never as a template. No resource of the module, read as the mesh reads it, +// carries content naming a host, and the raw module.json names none either. +func TestNoResourceCarriesContentNamingAHost(t *testing.T) { + host := regexp.MustCompile(`(?i)://[a-z0-9]|\b[a-z0-9-]+(\.[a-z0-9-]+)*\.(com|org|net|be|io|dev)\b`) + m := readManifest(t) + for i, r := range m.Resources { + if c, _ := r["content"].(string); host.MatchString(c) { + t.Errorf("resources[%d] (%v) carries content naming %q", i, r["id"], host.FindString(c)) + } + } + raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if h := host.FindString(string(raw)); h != "" { + t.Errorf("module.json names %q", h) + } +} + +// 1488: the toolchain's package.json and lock are shipped in the archive artifact `toolchain` and unpacked into the +// home's toolchain directory, where the program reads them. +func TestTheToolchainFilesAreAnArchiveUnpackedIntoTheHome(t *testing.T) { + m := readManifest(t) + r := resource(t, m, "toolchain-dir") + if r["type"] != "archive" || r["path"] != "${dir:state}/toolchain" || r["owner"] != "mesh-build-node" { + t.Errorf("toolchain-dir = %v, want an archive unpacked at ${dir:state}/toolchain", r) + } + if _, err := os.Stat(filepath.Join(moduleDir, "toolchain", "package-lock.json")); err != nil { + t.Errorf("the module carries no toolchain/package-lock.json: %v", err) + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module.go b/modules/mesh-build-node/cmd/mesh-build-node/module.go new file mode 100644 index 00000000..f237ca62 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/module.go @@ -0,0 +1,199 @@ +package main + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// CheckRound is how often the module reads the installed toolchain again, beside the check before every step and +// the one every health look makes. +const CheckRound = 10 * time.Minute + +// Module is the Node build module's one running state: the node it runs on, its recipe, the toolchain guard, the +// toolchain directory and the states it writes (toolchains, steps). +type Module struct { + Node string + // Recipe is the recipe in force. Reload, when set, reads it again at every check (the lock file the mesh + // writes can change under a running module); a failure to read it is a refusal. + Recipe Recipe + Reload func() (Recipe, error) + // Probe reads the installed Node version (ProbeNode). + Probe func(ctx context.Context) (string, error) + // Toolchain installs the toolchain directory; nil in a module that only guards. + Toolchain *Toolchain + // Work is the work directory, outside the home: a workspace is //tree. Run runs the commands of a + // step (a fake in a test). Clock is the time the steps are timed by. + Work string + Run forgekey.Runner + Clock func() time.Time + // Toolchains and Steps are the Put of the states of those names. + Toolchains, Steps buildstep.Put + // Emit tells the bus an event (stdio.Emit); nil emits nothing. The guard emits toolchain.reported after each write of + // the toolchains key, so the indexer, which holds no timer, learns that the bucket exists (issue 1444). + Emit func(eventType string, body any) error + // Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not). + Ctx context.Context + Say func(format string, a ...any) + + mu sync.Mutex // guards Recipe + install sync.Mutex // one install at a time +} + +func (m *Module) recipe() Recipe { + m.mu.Lock() + defer m.mu.Unlock() + return m.Recipe +} + +func (m *Module) guard(r Recipe) buildstep.Guard { + return buildstep.Guard{Language: "typescript", Want: r.Node, Recipe: r.Version(), Probe: m.Probe, Node: m.Node, Put: m.reportingPut()} +} + +// Check reads the installed toolchain, writes .typescript of the toolchains state and answers nil, or the +// refusal (buildstep.Guard.Check). It runs at start, before every step, at every health look and every CheckRound. +func (m *Module) Check(ctx context.Context) error { + if m.Reload != nil { + r, err := m.Reload() + if err != nil { + return &buildstep.Refusal{Text: err.Error(), Want: RecipeNode} + } + m.mu.Lock() + m.Recipe = r + m.mu.Unlock() + } + return m.guard(m.recipe()).Check(ctx) +} + +// Prepare is Check and then the toolchain directory of the recipe in force: what every step runs behind. +func (m *Module) Prepare(ctx context.Context) error { + if err := m.Check(ctx); err != nil { + return err + } + if m.Toolchain == nil { + return nil + } + m.install.Lock() + defer m.install.Unlock() + return m.Toolchain.Ensure(ctx, m.recipe().Version()) +} + +// Start prepares once, as the module starts, and then every CheckRound until ctx ends. +func (m *Module) Start(ctx context.Context) { + for { + if err := m.Prepare(ctx); err != nil && m.Say != nil { + m.Say("the toolchain: %v", err) + } + select { + case <-ctx.Done(): + return + case <-time.After(CheckRound): + } + } +} + +// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy while the toolchain installed is not +// the recipe's, saying both versions, so the controller raises the node's condition. It looks again each time it is +// asked. +func (m *Module) Health(ctx context.Context) health.Answer { + if err := m.Check(ctx); err != nil { + return health.Unhealthy(err.Error()) + } + r := m.recipe() + return health.Healthy(fmt.Sprintf("node %s installed, as the recipe %s builds with", r.Node, r.Version())) +} + +// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on +// its key in the steps state. The guard runs inside the step, before run, so a node that refuses ends every step +// refused with both versions named. +func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) { + if err := m.Prepare(ctx); err != nil { + return nil, err + } + return run(ctx, args) + }) + if err != nil { + return nil, err + } + key, _ := delivery.StepKey(m.Node, id) + return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil + } +} + +// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error. +func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + if err := m.Prepare(m.Ctx); err != nil { + return nil, err + } + return run(m.Ctx, args) + } +} + +// Status is what the module reports of itself: the node, the recipe, the toolchain found and why it refuses, if it +// does. +func (m *Module) Status(ctx context.Context) map[string]any { + err := m.Check(ctx) + installed, _ := m.Probe(ctx) + r := m.recipe() + out := map[string]any{"node": m.Node, "recipe": r.Version(), "node-version": r.Node, "installed": installed, "refusing": err != nil} + if m.Toolchain != nil { + out["toolchain-dir"] = m.Toolchain.Paths.Dir(r.Version()) + out["toolchain-installed"] = m.Toolchain.Present(r.Version()) + } + if err != nil { + out["why"] = err.Error() + } + return out +} + +func (m *Module) now() time.Time { + if m.Clock != nil { + return m.Clock() + } + return time.Now() +} + +// defaultPath is where a step looks for node, npm, git and ssh when this process was given no PATH. +const defaultPath = "/usr/local/bin:/usr/bin:/bin" + +// baseEnv is the environment of a step's command beyond what the step adds: PATH, the account's own HOME (its ssh +// files and git configuration) and a fixed locale. A command is given these and its own words, never this process's +// environment, which holds the module's bus credentials (ADR 0466 §4). +func (m *Module) baseEnv() []string { + return []string{"PATH=" + pathOrDefault(), "HOME=" + m.Toolchain.homeOrEmpty(), "LC_ALL=C"} +} + +func pathOrDefault() string { + if path := os.Getenv("PATH"); path != "" { + return path + } + return defaultPath +} + +// ToolchainReported is the event the guard emits after it writes the toolchains key. +const ToolchainReported = "toolchain.reported" + +// reportingPut is the toolchains Put that then emits toolchain.reported with the node and the language. A write that +// failed emits nothing; an emit that failed is said and does not refuse the step. +func (m *Module) reportingPut() buildstep.Put { + return func(key string, value any) (uint64, error) { + rev, err := m.Toolchains(key, value) + if err != nil || m.Emit == nil { + return rev, err + } + if eerr := m.Emit(ToolchainReported, map[string]string{"node": m.Node, "language": "typescript"}); eerr != nil && m.Say != nil { + m.Say("the event %s could not be emitted: %v", ToolchainReported, eerr) + } + return rev, nil + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module_test.go b/modules/mesh-build-node/cmd/mesh-build-node/module_test.go new file mode 100644 index 00000000..a698a184 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/module_test.go @@ -0,0 +1,231 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// keptStates stands in for the toolchains and the steps states. +type keptStates struct { + mu sync.Mutex + toolchains map[string]delivery.Toolchain + steps map[string][]delivery.Step + ended chan string +} + +func newStates() *keptStates { + return &keptStates{toolchains: map[string]delivery.Toolchain{}, steps: map[string][]delivery.Step{}, ended: make(chan string, 16)} +} + +func (s *keptStates) putToolchain(key string, value any) (uint64, error) { + tc, err := delivery.DecodeToolchain(value.(json.RawMessage)) + if err != nil { + return 0, err + } + s.mu.Lock() + defer s.mu.Unlock() + s.toolchains[key] = tc + return 1, nil +} + +func (s *keptStates) putStep(key string, value any) (uint64, error) { + st, err := delivery.DecodeStep(value.(json.RawMessage)) + if err != nil { + return 0, err + } + s.mu.Lock() + s.steps[key] = append(s.steps[key], st) + s.mu.Unlock() + if st.State != delivery.StepRunning { + s.ended <- key + } + return 1, nil +} + +func (s *keptStates) toolchain(key string) (delivery.Toolchain, bool) { + s.mu.Lock() + defer s.mu.Unlock() + tc, ok := s.toolchains[key] + return tc, ok +} + +func (s *keptStates) waitEnd(t *testing.T) delivery.Step { + t.Helper() + select { + case key := <-s.ended: + s.mu.Lock() + defer s.mu.Unlock() + v := s.steps[key] + return v[len(v)-1] + case <-time.After(3 * time.Second): + t.Fatal("the step never ended on its key") + return delivery.Step{} + } +} + +func moduleWith(states *keptStates, installed string) *Module { + return &Module{Node: "ace", Recipe: Recipe{Steps: "1", Node: "v26.10.0", Lock: "lock"}, + Probe: func(context.Context) (string, error) { return installed, nil }, + Toolchains: states.putToolchain, Steps: states.putStep, Ctx: context.Background()} +} + +// 1488/AC3: a probe answering v26.10.0: the bundle's start writes ace.typescript with that version and the recipe's, +// and the health tool answers healthy. +func TestStartWritesTheToolchainAndHealthIsHealthy(t *testing.T) { + states := newStates() + m := moduleWith(states, "v26.10.0") + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go m.Start(ctx) + var tc delivery.Toolchain + for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + if got, ok := states.toolchain("ace.typescript"); ok { + tc = got + break + } + } + if tc.Version != "v26.10.0" || tc.Recipe != m.Recipe.Version() || tc.Language != "typescript" { + t.Fatalf("ace.typescript = %+v, want v26.10.0 and recipe %s", tc, m.Recipe.Version()) + } + if a := m.Health(ctx); a.State != health.StateHealthy || !a.Healthy { + t.Errorf("health = %+v, want healthy", a) + } +} + +// 1488/AC3: a probe answering v26.11.0: every step tool's step ends refused naming both versions, ace.typescript +// holds v26.11.0, and the health tool answers unhealthy naming both. +func TestAnotherNodeRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) { + states := newStates() + m := moduleWith(states, "v26.11.0") + ran := false + run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil } + steps := append([]StepTool{ + {Name: "long_probe", Long: true, Run: run}, + {Name: "short_probe", Run: run}, + }, (&Module{}).StepTools()...) + for _, tool := range m.Tools(steps) { + if tool.Name == StatusTool || tool.Name == HealthTool { + continue + } + var reason string + out, err := tool.Run(map[string]any{}) + if isLong(steps, tool.Name) { + if err != nil { + t.Fatalf("%s: %v", tool.Name, err) + } + if out.(map[string]any)["step"] == "" { + t.Fatalf("%s answered no step id: %v", tool.Name, out) + } + st := states.waitEnd(t) + if st.State != delivery.StepRefused { + t.Errorf("%s: the step ended %s, want refused", tool.Name, st.State) + } + reason = st.Reason + } else { + if err == nil { + t.Fatalf("%s: answered %v, want a refusal", tool.Name, out) + } + reason = err.Error() + } + if !strings.Contains(reason, "v26.11.0") || !strings.Contains(reason, "v26.10.0") { + t.Errorf("%s: the refusal %q does not name both versions", tool.Name, reason) + } + } + if ran { + t.Errorf("a step ran on a node that refuses") + } + if tc, _ := states.toolchain("ace.typescript"); tc.Version != "v26.11.0" { + t.Errorf("ace.typescript holds %q, want the installed v26.11.0", tc.Version) + } + a := m.Health(context.Background()) + if a.State != health.StateUnhealthy || !strings.Contains(a.Why, "v26.11.0") || !strings.Contains(a.Why, "v26.10.0") { + t.Errorf("health = %+v, want unhealthy naming both versions", a) + } +} + +func isLong(steps []StepTool, name string) bool { + for _, s := range steps { + if s.Name == name { + return s.Long + } + } + return false +} + +// 1488/AC2: the recipe version moves with one byte of the lock and with the Node version, and with the recipe's own +// version; equal recipes have the same one. +func TestRecipeVersionMovesWithEachOfItsValues(t *testing.T) { + lock := []byte(`{"lockfileVersion":3}`) + base := Recipe{Steps: "1", Node: "v26.10.0", Lock: LockHash(lock)} + if base.Version() != (Recipe{Steps: "1", Node: "v26.10.0", Lock: LockHash(lock)}).Version() { + t.Errorf("one recipe has two versions") + } + seen := map[string]string{base.Version(): "base"} + for name, r := range map[string]Recipe{ + "one byte of the lock": {Steps: "1", Node: "v26.10.0", Lock: LockHash([]byte(`{"lockfileVersion":4}`))}, + "node": {Steps: "1", Node: "v26.11.0", Lock: LockHash(lock)}, + "steps": {Steps: "2", Node: "v26.10.0", Lock: LockHash(lock)}, + } { + v := r.Version() + if was, dup := seen[v]; dup { + t.Errorf("the recipe differing in %s has the version of the %s recipe: %s", name, was, v) + } + seen[v] = name + } +} + +// 1488/AC2: the recipe reads the lock file's bytes: a lock changed on disk is another recipe version, a lock missing +// is an error. +func TestReadRecipeHashesTheLockFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "package-lock.json") + if _, err := ReadRecipe(path); err == nil { + t.Errorf("a missing lock was accepted") + } + os.WriteFile(path, []byte("a"), 0o600) + a, err := ReadRecipe(path) + if err != nil || a.Node != "v26.10.0" { + t.Fatalf("= %+v, %v", a, err) + } + os.WriteFile(path, []byte("b"), 0o600) + b, _ := ReadRecipe(path) + if a.Version() == b.Version() { + t.Errorf("one byte of the lock did not move the recipe version") + } +} + +// 1488/AC3: a lock changed under a running module is read at the next check, and a lock that cannot be read refuses. +func TestACheckReadsTheRecipeAgain(t *testing.T) { + states := newStates() + m := moduleWith(states, "v26.10.0") + r2 := Recipe{Steps: "1", Node: "v26.10.0", Lock: "other"} + m.Reload = func() (Recipe, error) { return r2, nil } + if err := m.Check(context.Background()); err != nil { + t.Fatal(err) + } + if tc, _ := states.toolchain("ace.typescript"); tc.Recipe != r2.Version() { + t.Errorf("ace.typescript holds recipe %s, want the reloaded %s", tc.Recipe, r2.Version()) + } + m.Reload = func() (Recipe, error) { return Recipe{}, errors.New("no lock") } + if err := m.Check(context.Background()); err == nil || m.Health(context.Background()).Healthy { + t.Errorf("a lock that cannot be read neither refused nor made the module unhealthy: %v", err) + } +} + +func TestParseNodeVersion(t *testing.T) { + if v, err := parseNodeVersion("v26.10.0\n"); err != nil || v != "v26.10.0" { + t.Errorf("= %q, %v", v, err) + } + if _, err := parseNodeVersion("not node"); err == nil { + t.Errorf("accepted a text that is not `node --version`'s") + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/probe.go b/modules/mesh-build-node/cmd/mesh-build-node/probe.go new file mode 100644 index 00000000..e2f99dff --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/probe.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "fmt" + "os/exec" + "strings" + "time" +) + +// ProbeTimeout bounds `node --version`. +const ProbeTimeout = 15 * time.Second + +// ProbeNode reads the installed Node version: what `node --version` prints ("v26.10.0"), as RecipeNode is written. +func ProbeNode(ctx context.Context) (string, error) { + ctx, cancel := context.WithTimeout(ctx, ProbeTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, "node", "--version").Output() + if err != nil { + return "", err + } + return parseNodeVersion(string(out)) +} + +func parseNodeVersion(out string) (string, error) { + v := strings.TrimSpace(out) + if len(v) < 2 || v[0] != 'v' || v[1] < '0' || v[1] > '9' || strings.ContainsAny(v, " \n\t") { + return "", fmt.Errorf("`node --version` answered %q", v) + } + return v, nil +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/recipe.go b/modules/mesh-build-node/cmd/mesh-build-node/recipe.go new file mode 100644 index 00000000..b024df00 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/recipe.go @@ -0,0 +1,58 @@ +package main + +// The recipe of the Node build module (novox/hq ADR 0405 §1 and §7, ADR 0469 §1 and §4; issue 1488): the one Node +// version it builds with, and the recipe's own version, a hash of the recipe's version, that Node version and the +// bytes of the lock file that names the TypeScript toolchain. A node whose installed Node differs from the recipe's +// refuses every step (module.go), so all builders run this version or none builds. + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" +) + +// RecipeNode is the Node version the recipe builds with: the text `node --version` prints for it, which is what the +// guard compares and what the indexer reads from .typescript of the toolchains state. +const RecipeNode = "v26.10.0" + +// RecipeSteps is the recipe's own version: changed whenever the steps of fetch, test or build change in a way that +// can change what they produce (ADR 0405 §7). It is one of the three values the recipe version hashes. +const RecipeSteps = "1" + +// Recipe is every value that makes up the recipe version. +type Recipe struct { + // Steps is RecipeSteps. + Steps string + // Node is the toolchain's version as `node --version` prints it. + Node string + // Lock is the hex sha256 of the bytes of toolchain/package-lock.json (LockHash). + Lock string +} + +// LockHash is the text a lock file's bytes stand for in the recipe version. +func LockHash(lock []byte) string { + sum := sha256.Sum256(lock) + return hex.EncodeToString(sum[:]) +} + +// ReadRecipe is the recipe of this build of the module with the lock file read from path. A missing lock is an +// error: with no lock there is no toolchain to name. +func ReadRecipe(lockPath string) (Recipe, error) { + raw, err := os.ReadFile(lockPath) + if err != nil { + return Recipe{}, fmt.Errorf("the toolchain's lock file cannot be read: %w", err) + } + return Recipe{Steps: RecipeSteps, Node: RecipeNode, Lock: LockHash(raw)}, nil +} + +// Version is the recipe version: a hash of the recipe's own version, the Node version and the lock file's hash, so a +// change of any of the three moves it and with it every fingerprint that holds it (ADR 0405 §7, ADR 0465 §2). The +// fields are named in the hashed text, so no two recipes share a text. +func (r Recipe) Version() string { + h := sha256.New() + for _, f := range [][2]string{{"steps", r.Steps}, {"node", r.Node}, {"lock", r.Lock}} { + h.Write([]byte(f[0] + "=" + f[1] + "\n")) + } + return hex.EncodeToString(h.Sum(nil))[:16] +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/registry.go b/modules/mesh-build-node/cmd/mesh-build-node/registry.go new file mode 100644 index 00000000..b4a2bc66 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/registry.go @@ -0,0 +1,81 @@ +package main + +// How the toolchain's install reaches the mesh's package registry (novox/hq ADR 0405 §5, ADR 0336; issue 1488). +// The lock names @novox/mesh-sdk at the registry's public address, and npm installs a locked tarball only from the +// registry it is configured with, so the install is told that registry for the @novox scope and nothing else: every +// other package resolves from the public one. This is how today's build agent names it (mesh-controller +// internal/builder/dependencies.go: the scope's registry from the npm-package-registry binding, the account and +// secret the mesh sealed), read here from the module's own binding and secret files (module.json `binds` and +// `secrets`), the same files the lab module reads. + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "strings" +) + +// Scope is the npm scope the mesh's registry answers for. +const Scope = "@novox" + +// Registry is the mesh's package registry as the binding names it. +type Registry struct { + URL, Username, Password string +} + +// ReadRegistry reads the npm-package-registry binding and its sealed secret. The registry is the binding's +// `public-root` when it has one (the address the lock holds), and otherwise scheme://at:port plus `npm-path`. +func ReadRegistry(bindingPath, secretPath string) (Registry, error) { + raw, err := os.ReadFile(bindingPath) + if err != nil { + return Registry{}, fmt.Errorf("no package registry is bound to this module: %w", err) + } + var b struct { + At string `json:"at"` + As string `json:"as"` + Serves struct { + Scheme string `json:"scheme"` + Port int `json:"port"` + NpmPath string `json:"npm-path"` + PublicRoot string `json:"public-root"` + } `json:"serves"` + } + if err := json.Unmarshal(raw, &b); err != nil { + return Registry{}, fmt.Errorf("the package registry binding is not JSON: %w", err) + } + url := b.Serves.PublicRoot + if url == "" && b.At != "" && b.Serves.NpmPath != "" { + scheme := b.Serves.Scheme + if scheme == "" { + scheme = "http" + } + url = fmt.Sprintf("%s://%s:%d%s", scheme, b.At, b.Serves.Port, b.Serves.NpmPath) + } + if url == "" { + return Registry{}, fmt.Errorf("the package registry binding names no address") + } + if !strings.HasSuffix(url, "/") { + url += "/" + } + secret, err := os.ReadFile(secretPath) + if err != nil { + return Registry{}, fmt.Errorf("the package registry's secret cannot be read: %w", err) + } + return Registry{URL: url, Username: b.As, Password: strings.TrimSpace(string(secret))}, nil +} + +// Npmrc is the npm configuration text for the install: the @novox scope at the registry, with the account's +// credential for it when there is one. +func (r Registry) Npmrc() string { + var sb strings.Builder + fmt.Fprintf(&sb, "%s:registry=%s\n", Scope, r.URL) + if r.Username != "" && r.Password != "" { + hostPath := r.URL + if i := strings.Index(hostPath, "://"); i >= 0 { + hostPath = hostPath[i+1:] + } + fmt.Fprintf(&sb, "%s:_auth=%s\n", hostPath, base64.StdEncoding.EncodeToString([]byte(r.Username+":"+r.Password))) + } + return sb.String() +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go b/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go new file mode 100644 index 00000000..2feaac95 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go @@ -0,0 +1,106 @@ +package main + +// The toolchain directory (novox/hq ADR 0405 §5, ADR 0469 §4; issue 1488): typescript, esbuild, @novox/mesh-sdk and +// nats by the versions the module's own lock names, installed by `npm ci --ignore-scripts` into +// /toolchain/ when that directory is missing. No bin links are made (--no-bin-links): the +// toolchain holds no symlink, and tsc and esbuild are run by their real paths. A change of the lock, of the Node +// version or of the recipe moves the recipe version and so installs a fresh directory beside the old one; steps use the one named +// by the current recipe. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// installedMark is written last into a toolchain directory, so a directory an install did not finish is not taken +// for a toolchain. +const installedMark = ".installed" + +// Paths are the module's places on its machine. +type Paths struct { + // Home is the account's home (the state directory): where the mesh writes toolchain/package.json and + // toolchain/package-lock.json (module.json resources) and the registry binding and secret. + Home string + // Cache is the cache directory, outside the home (ADR 0466 §1). + Cache string +} + +// LockPath is the committed lock file the mesh writes into the home. +func (p Paths) LockPath() string { return filepath.Join(p.Home, "toolchain", "package-lock.json") } + +// Dir is the toolchain directory of a recipe version. +func (p Paths) Dir(version string) string { return filepath.Join(p.Cache, "toolchain", version) } + +// Toolchain installs the toolchain directories. +type Toolchain struct { + Paths Paths + Run forgekey.Runner + Registry func() (Registry, error) +} + +// Present reports whether the recipe version's toolchain directory is installed. +func (t Toolchain) Present(version string) bool { + _, err := os.Stat(filepath.Join(t.Paths.Dir(version), installedMark)) + return err == nil +} + +// Ensure installs the recipe version's toolchain directory unless it is there: the module's package.json and lock +// are copied in, and `npm ci --ignore-scripts` runs once, the mesh's registry named for the @novox scope. A directory +// an earlier install left unfinished is removed first. +func (t Toolchain) Ensure(ctx context.Context, version string) error { + if t.Present(version) { + return nil + } + dir := t.Paths.Dir(version) + if err := os.RemoveAll(dir); err != nil { + return err + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + for _, name := range []string{"package.json", "package-lock.json"} { + raw, err := os.ReadFile(filepath.Join(t.Paths.Home, "toolchain", name)) + if err != nil { + return fmt.Errorf("the toolchain's %s cannot be read: %w", name, err) + } + if err := os.WriteFile(filepath.Join(dir, name), raw, 0o600); err != nil { + return err + } + } + reg, err := t.Registry() + if err != nil { + return err + } + npmrc := filepath.Join(dir, ".npmrc-install") + if err := os.WriteFile(npmrc, []byte(reg.Npmrc()), 0o600); err != nil { + return err + } + defer os.Remove(npmrc) + ran := t.Run(ctx, []string{"PATH=" + pathOrDefault(), "HOME=" + t.Paths.Home, "LC_ALL=C", "npm_config_cache=" + filepath.Join(t.Paths.Cache, "npm")}, + "npm", "ci", "--ignore-scripts", "--no-bin-links", "--no-audit", "--no-fund", "--prefix", dir, "--userconfig", npmrc) + if ran.Err != "" || ran.Status != 0 { + return fmt.Errorf("npm ci --ignore-scripts for the toolchain %s failed (status %d %s): %s", + version, ran.Status, ran.Err, tail(ran.Stderr)) + } + return os.WriteFile(filepath.Join(dir, installedMark), []byte(version+"\n"), 0o600) +} + +func tail(s string) string { + s = strings.TrimSpace(s) + if len(s) > 400 { + s = s[len(s)-400:] + } + return s +} + +func (t *Toolchain) homeOrEmpty() string { + if t == nil { + return "" + } + return t.Paths.Home +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go b/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go new file mode 100644 index 00000000..618bc421 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +type npmCall struct { + name string + args []string + env []string +} + +type fakeNpm struct { + mu sync.Mutex + calls []npmCall + ran forgekey.Ran +} + +func (f *fakeNpm) run(_ context.Context, env []string, name string, args ...string) forgekey.Ran { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, npmCall{name, args, env}) + if f.ran.Status == 0 && f.ran.Err == "" { + // npm ci writes node_modules into --prefix. + for i, a := range args { + if a == "--prefix" { + os.MkdirAll(filepath.Join(args[i+1], "node_modules"), 0o700) + } + } + } + return f.ran +} + +func toolchainIn(t *testing.T, fake *fakeNpm) Toolchain { + t.Helper() + home, cache := t.TempDir(), t.TempDir() + if err := os.MkdirAll(filepath.Join(home, "toolchain"), 0o700); err != nil { + t.Fatal(err) + } + os.WriteFile(filepath.Join(home, "toolchain", "package.json"), []byte(`{"name":"x"}`), 0o600) + os.WriteFile(filepath.Join(home, "toolchain", "package-lock.json"), []byte(`{"lock":1}`), 0o600) + return Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: fake.run, + Registry: func() (Registry, error) { + return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil + }} +} + +// 1488/AC4: given a missing toolchain directory, npm ci --ignore-scripts runs once in /toolchain/ with the module's lock; asked again with the directory present, nothing runs. +func TestAMissingToolchainDirectoryIsInstalledOnceFromTheLock(t *testing.T) { + fake := &fakeNpm{} + tc := toolchainIn(t, fake) + version := Recipe{Steps: "1", Node: "v26.10.0", Lock: "x"}.Version() + dir := filepath.Join(tc.Paths.Cache, "toolchain", version) + if tc.Present(version) { + t.Fatal("present before any install") + } + if err := tc.Ensure(context.Background(), version); err != nil { + t.Fatal(err) + } + if len(fake.calls) != 1 { + t.Fatalf("npm ran %d times, want once", len(fake.calls)) + } + c := fake.calls[0] + joined := strings.Join(c.args, " ") + if c.name != "npm" || c.args[0] != "ci" || !strings.Contains(joined, "--ignore-scripts") || !strings.Contains(joined, "--no-bin-links") || !strings.Contains(joined, "--prefix "+dir) { + t.Errorf("ran %s %v, want npm ci --ignore-scripts --prefix %s", c.name, c.args, dir) + } + if lock, _ := os.ReadFile(filepath.Join(dir, "package-lock.json")); string(lock) != `{"lock":1}` { + t.Errorf("the directory holds lock %q, want the module's", lock) + } + if _, err := os.Stat(filepath.Join(dir, ".npmrc-install")); err == nil { + t.Errorf("the registry credential file was left in the toolchain directory") + } + if !tc.Present(version) { + t.Errorf("not present after the install") + } + if err := tc.Ensure(context.Background(), version); err != nil || len(fake.calls) != 1 { + t.Errorf("a present directory ran npm again: %d calls, %v", len(fake.calls), err) + } +} + +// 1488/AC4: the install is told the mesh's registry for @novox with the account's credential, and only through a +// config file of its own, never on the command line. +func TestTheInstallNamesTheRegistryInAConfigFileNotOnTheCommandLine(t *testing.T) { + var npmrc string + fake := &fakeNpm{} + tc := toolchainIn(t, fake) + tc.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + for i, a := range args { + if a == "--userconfig" { + raw, _ := os.ReadFile(args[i+1]) + npmrc = string(raw) + } + } + return fake.run(ctx, env, name, args...) + } + if err := tc.Ensure(context.Background(), "v1"); err != nil { + t.Fatal(err) + } + if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n") || + !strings.Contains(npmrc, "//git.novox.be/api/packages/novox/npm/:_auth=") { + t.Errorf("npmrc = %q", npmrc) + } + if strings.Contains(strings.Join(fake.calls[0].args, " "), "pw") { + t.Errorf("the credential is on the command line: %v", fake.calls[0].args) + } +} + +// 1488/AC4: an install that fails leaves no directory taken for a toolchain, says why, and the next call tries again. +func TestAFailedInstallIsNotAToolchain(t *testing.T) { + fake := &fakeNpm{ran: forgekey.Ran{Status: 1, Stderr: "E401 unauthorized"}} + tc := toolchainIn(t, fake) + err := tc.Ensure(context.Background(), "v1") + if err == nil || !strings.Contains(err.Error(), "E401") { + t.Fatalf("err = %v, want the npm failure named", err) + } + if tc.Present("v1") { + t.Errorf("a failed install is present") + } + fake.ran = forgekey.Ran{} + if err := tc.Ensure(context.Background(), "v1"); err != nil || len(fake.calls) != 2 || !tc.Present("v1") { + t.Errorf("the retry: %v, %d calls", err, len(fake.calls)) + } +} + +// 1488/AC4: a module's steps run the install behind the guard: a refusing node installs nothing, an accepting one +// installs once however many steps follow. +func TestPrepareInstallsOnlyBehindAPassingGuard(t *testing.T) { + fake := &fakeNpm{} + tc := toolchainIn(t, fake) + states := newStates() + m := moduleWith(states, "v26.11.0") + m.Toolchain = &tc + if err := m.Prepare(context.Background()); err == nil || len(fake.calls) != 0 { + t.Fatalf("a refusing node: %v, %d npm calls", err, len(fake.calls)) + } + m = moduleWith(states, "v26.10.0") + m.Toolchain = &tc + for i := 0; i < 3; i++ { + if err := m.Prepare(context.Background()); err != nil { + t.Fatal(err) + } + } + if len(fake.calls) != 1 { + t.Errorf("npm ran %d times for three steps, want once", len(fake.calls)) + } +} + +func TestRegistryFromTheBinding(t *testing.T) { + dir := t.TempDir() + b, s := filepath.Join(dir, "b.json"), filepath.Join(dir, "s") + os.WriteFile(s, []byte("secret\n"), 0o600) + os.WriteFile(b, []byte(`{"provision":"npm-package-registry","at":"10.42.0.1","as":"mesh-build-node-ace","serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`), 0o600) + r, err := ReadRegistry(b, s) + if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Username != "mesh-build-node-ace" || r.Password != "secret" { + t.Fatalf("= %+v, %v", r, err) + } + os.WriteFile(b, []byte(`{"at":"h","as":"a","serves":{"port":1,"npm-path":"/p/","public-root":"https://git.novox.be/p"}}`), 0o600) + if r, _ := ReadRegistry(b, s); r.URL != "https://git.novox.be/p/" { + t.Errorf("the public root is not preferred: %q", r.URL) + } + if _, err := ReadRegistry(filepath.Join(dir, "none"), s); err == nil { + t.Errorf("no binding was accepted") + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/tools.go b/modules/mesh-build-node/cmd/mesh-build-node/tools.go new file mode 100644 index 00000000..5d3cf589 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/tools.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The tools this module serves. A tool is named as the module is: build_node_ and its verb (a tool name has no dot). +const ( + StatusTool = "build_node_status" + HealthTool = "build_node_health" +) + +// StepTool is one step tool of the module: fetch, test or build. Long is a step that answers at once and +// ends on its key (ADR 0469 §2); the others answer in their call. Both run behind the toolchain guard. +type StepTool struct { + Name string + Description string + Input map[string]any + Long bool + Run func(ctx context.Context, args map[string]any) (any, error) +} + +// StepTools are the module's step tools. Each step's issue adds its own. +func (m *Module) StepTools() []StepTool { + return []StepTool{m.fetchTool(), m.testTool(), m.buildTool()} +} + +// Tools is every tool the module serves: its step tools behind the guard, its status and its health. +func (m *Module) Tools(steps []StepTool) []stdio.Tool { + tools := []stdio.Tool{} + for _, s := range steps { + t := stdio.Tool{Name: s.Name, Description: s.Description, Input: s.Input} + if s.Long { + t.Run = m.Long(s.Name, s.Run) + } else { + t.Run = m.Short(s.Run) + } + tools = append(tools, t) + } + return append(tools, + stdio.Tool{Name: StatusTool, + Description: "What the Node build module is on this machine: the Node version its recipe builds with and the one installed, " + + "the recipe version, the toolchain directory and whether it is installed, and why it refuses to build if it does. Read-only.", + Run: func(map[string]any) (any, error) { return m.Status(m.Ctx), nil }}, + stdio.Tool{Name: HealthTool, + Description: "How the Node build module is: unhealthy while the Node installed on this machine is not the version its " + + "recipe builds with, saying both versions; healthy otherwise. Read-only.", + Run: func(map[string]any) (any, error) { return m.Health(m.Ctx), nil }}, + ) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/workspace.go b/modules/mesh-build-node/cmd/mesh-build-node/workspace.go new file mode 100644 index 00000000..001dec70 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/workspace.go @@ -0,0 +1,187 @@ +package main + +// A workspace and the packages a step runs against (novox/hq ADR 0469 §2 and §5; issues 1490, 1491). A workspace is +// //tree, made by the fetch step. The toolchain's packages are copied to // +// node_modules, which lies above the tree: node, tsc and esbuild resolve an import by walking up from the importing +// file, so @novox/mesh-sdk and the types are found there while the tree itself, which a bundle is packed from, stays +// what the commit holds. A copy and not a link, because the mesh makes no symlink. + +import ( + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +var workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`) + +// notCopied are the toolchain's own tools: a bundle imports none of them, and they are tens of megabytes. +var notCopied = map[string]bool{"typescript": true, "esbuild": true, "@esbuild": true} + +// moduleDirOf answers the directory of a module in a workspace and the workspace's root (the step's directory), +// refusing an id or a directory that is not what a fetch made. +func (m *Module) moduleDirOf(workspace, module string) (root, dir string, err error) { + if !workspaceOK.MatchString(workspace) { + return "", "", fmt.Errorf("%q is not a workspace id", workspace) + } + if module == "" { + module = "." + } + clean := filepath.Clean(filepath.FromSlash(module)) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", "", fmt.Errorf("the module directory %q is not inside the workspace", module) + } + root = filepath.Join(m.Work, workspace) + dir = filepath.Join(root, "tree", clean) + if info, err := os.Stat(dir); err != nil || !info.IsDir() { + return "", "", fmt.Errorf("workspace %s holds no directory %q", workspace, module) + } + return root, dir, nil +} + +// toolchainDir is the installed toolchain directory of the recipe in force. +func (m *Module) toolchainDir() string { + return m.Toolchain.Paths.Dir(m.recipe().Version()) +} + +// tsc and esbuild are the toolchain's own programs, run by their real paths. +func (m *Module) tsc() string { + return filepath.Join(m.toolchainDir(), "node_modules", "typescript", "bin", "tsc") +} +func (m *Module) esbuild() string { + return filepath.Join(m.toolchainDir(), "node_modules", "esbuild", "bin", "esbuild") +} + +// provide copies the toolchain's packages to /node_modules unless they are there. +func (m *Module) provide(root string) error { + if err := m.writeBin(filepath.Join(root, "bin")); err != nil { + return err + } + dest := filepath.Join(root, "node_modules") + if _, err := os.Stat(dest); err == nil { + return nil + } + return copyPackages(filepath.Join(m.toolchainDir(), "node_modules"), dest) +} + +// binDir is the workspace's bin directory: small shell scripts, files and not links, that call the toolchain's tsc and +// esbuild, so a package's scripts (`tsc`, `esbuild` by name) find them. npm ci runs with --no-bin-links, which leaves +// the toolchain no .bin directory of its own. +func binDir(root string) string { return filepath.Join(root, "bin") } + +func (m *Module) writeBin(dir string) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + for name, body := range map[string]string{ + "tsc": "exec node '" + m.tsc() + "' \"$@\"\n", + "esbuild": "exec '" + m.esbuild() + "' \"$@\"\n", + } { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0o755); err != nil { + return err + } + if err := os.Chmod(path, 0o755); err != nil { + return err + } + } + return nil +} + +// copyTree copies a directory of regular files and directories, keeping permission bits, except a build's output and +// the repository's own .git. Anything else in it (a link) is an error: the mesh makes no symlink. +func copyTree(from, to string) error { + return filepath.WalkDir(from, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(from, path) + if d.IsDir() && (rel == ".mesh-build" || rel == ".git") { + return filepath.SkipDir + } + switch { + case d.IsDir(): + return os.MkdirAll(filepath.Join(to, rel), 0o755) + case d.Type().IsRegular(): + return copyFile(path, filepath.Join(to, rel)) + default: + return fmt.Errorf("%s is neither a file nor a directory, and the build copies only those", path) + } + }) +} + +// copyPackages copies the packages of a node_modules directory except the toolchain's own tools. It copies regular +// files and directories only and keeps their permission bits; anything else in the source is an error. +func copyPackages(from, to string) error { + if err := os.MkdirAll(to, 0o755); err != nil { + return err + } + return filepath.WalkDir(from, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(from, path) + if rel == "." { + return nil + } + if top := strings.Split(rel, string(filepath.Separator))[0]; notCopied[top] || top == ".package-lock.json" || top == ".bin" { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + switch { + case d.IsDir(): + return os.MkdirAll(filepath.Join(to, rel), 0o755) + case d.Type().IsRegular(): + return copyFile(path, filepath.Join(to, rel)) + default: + return fmt.Errorf("%s is neither a file nor a directory, and the packages copied carry only those", path) + } + }) +} + +func copyFile(from, to string) error { + in, err := os.Open(from) + if err != nil { + return err + } + defer in.Close() + info, err := in.Stat() + if err != nil { + return err + } + out, err := os.OpenFile(to, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm()) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() +} + +// runIn runs a command with dir as its working directory (`env -C`), so it is the plain line the build agent runs from +// the module's root, with the same relative paths. The command's environment is baseEnv and extra. +func (m *Module) runIn(ctx context.Context, dir string, extra []string, name string, args ...string) forgekey.Ran { + return m.Run(ctx, append(m.baseEnv(), extra...), "env", append([]string{"-C", dir, name}, args...)...) +} + +// ranFailed says why a command did not pass, or "" when it did. +func ranFailed(r forgekey.Ran) string { + if r.Status == 0 && r.Err == "" { + return "" + } + out := strings.TrimSpace(r.Stdout + "\n" + r.Stderr) + if len(out) > 1500 { + out = out[:1500] + " ..." + } + return fmt.Sprintf("status %d %s: %s", r.Status, r.Err, out) +} diff --git a/modules/mesh-build-node/files/forge b/modules/mesh-build-node/files/forge new file mode 100644 index 00000000..6bbf4e20 --- /dev/null +++ b/modules/mesh-build-node/files/forge @@ -0,0 +1,7 @@ +Host ${bound:forge-account:ssh_alias} + HostName ${bound:forge-account:ssh_host} + Port ${bound:forge-account:ssh_port} + HostKeyAlias ${bound:forge-account:ssh_alias} + User git + IdentityFile ~/.ssh/mesh_ed25519 + IdentitiesOnly yes diff --git a/modules/mesh-build-node/files/gitconfig b/modules/mesh-build-node/files/gitconfig new file mode 100644 index 00000000..05252302 --- /dev/null +++ b/modules/mesh-build-node/files/gitconfig @@ -0,0 +1,2 @@ +[url "ssh://${bound:forge-account:ssh_alias}/"] + insteadOf = ${bound:forge-account:https_base} diff --git a/modules/mesh-build-node/files/known_hosts b/modules/mesh-build-node/files/known_hosts new file mode 100644 index 00000000..e0df6d81 --- /dev/null +++ b/modules/mesh-build-node/files/known_hosts @@ -0,0 +1 @@ +${bound:forge-account:ssh_alias} ${bound:forge-account:host_key} diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod new file mode 100644 index 00000000..261771f9 --- /dev/null +++ b/modules/mesh-build-node/go.mod @@ -0,0 +1,5 @@ +module meshbuildnode + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum new file mode 100644 index 00000000..132a607b --- /dev/null +++ b/modules/mesh-build-node/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d h1:mmOCJphjQN15jNVh+sMQqnmJ3UiYltiuQRg/zw3wFtw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json new file mode 100644 index 00000000..812f7a0e --- /dev/null +++ b/modules/mesh-build-node/module.json @@ -0,0 +1,174 @@ +{ + "module": "mesh-build-node", + "version": "1", + "runs-as": "mesh-build-node", + "capabilities": [ + "package-manager", + "user-namespaces" + ], + "state": [ + "toolchains", + "keys", + "steps" + ], + "tools": [ + "build_node_fetch", + "build_node_test", + "build_node_build", + "build_node_status", + "build_node_health" + ], + "streams": [ + { + "name": "toolchains-node", + "events": [ + "toolchain.**" + ] + } + ], + "requires": [ + "forge-account", + "npm-package-registry" + ], + "contributes": { + "npm-package-registry": { + "access": "read" + } + }, + "binds": { + "npm-package-registry": "${dir:state}/package-registry.json" + }, + "secrets": { + "npm-package-registry": "${dir:state}/package-registry.secret" + }, + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-build-node", + "runtime-health": { + "kind": "tool", + "tool": "build_node_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + }, + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-build-node", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-build-node" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-build-node", + "place": ".", + "unknown": "program" + }, + { + "id": "work", + "type": "directory", + "path": "/var/lib/mesh-build-node-work", + "mode": "0700", + "owner": "mesh-build-node", + "unknown": "program" + }, + { + "id": "cache", + "type": "directory", + "path": "/var/lib/mesh-build-node-cache", + "mode": "0700", + "owner": "mesh-build-node", + "unknown": "program" + }, + { + "id": "ssh-dir", + "type": "directory", + "path": "${dir:state}/.ssh", + "mode": "0700", + "owner": "mesh-build-node", + "unknown": "program" + }, + { + "id": "ssh-config", + "type": "file", + "path": "${dir:ssh-dir}/config", + "owner": "mesh-build-node", + "mode": "0600", + "template": "files/forge" + }, + { + "id": "known-hosts", + "type": "file", + "path": "${dir:ssh-dir}/known_hosts", + "owner": "mesh-build-node", + "mode": "0644", + "template": "files/known_hosts" + }, + { + "id": "git-config", + "type": "file", + "path": "${dir:state}/.config/git/config", + "owner": "mesh-build-node", + "mode": "0644", + "template": "files/gitconfig" + }, + { + "id": "toolchain-dir", + "type": "archive", + "path": "${dir:state}/toolchain", + "owner": "mesh-build-node", + "artifact": "toolchain", + "unknown": "program" + }, + { + "id": "nodejs", + "type": "package", + "package": "nodejs" + }, + { + "id": "npm", + "type": "package", + "package": "npm" + }, + { + "id": "git", + "type": "package", + "package": "git" + }, + { + "id": "openssh", + "type": "package", + "package": "openssh" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-build-node", + "binary": "mesh-build-node", + "loads": [ + "mesh-build-node" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + }, + { + "name": "toolchain", + "kind": "archive", + "from": "toolchain" + } + ] + } +} diff --git a/modules/mesh-build-node/toolchain/package-lock.json b/modules/mesh-build-node/toolchain/package-lock.json new file mode 100644 index 00000000..e4867d9c --- /dev/null +++ b/modules/mesh-build-node/toolchain/package-lock.json @@ -0,0 +1,572 @@ +{ + "name": "mesh-build-node-toolchain", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "mesh-build-node-toolchain", + "version": "1.0.0", + "dependencies": { + "@novox/mesh-sdk": "0.1.14", + "nats": "2.29.3" + }, + "devDependencies": { + "@types/node": "22.20.1", + "esbuild": "0.25.12", + "typescript": "5.9.3" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@novox/mesh-sdk": { + "version": "0.1.14", + "resolved": "https://git.novox.be/api/packages/novox/npm/%40novox%2Fmesh-sdk/-/0.1.14/mesh-sdk-0.1.14.tgz", + "integrity": "sha512-ICdIfVicQQXSw6fkLiwzoRx3weiWh0jsIEYG/m98CKgFIlziZzb7V+ZmT/TvPO9bTDFDtfqDLoFu+OXY5+ryQw==" + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/nats": { + "version": "2.29.3", + "resolved": "https://registry.npmjs.org/nats/-/nats-2.29.3.tgz", + "integrity": "sha512-tOQCRCwC74DgBTk4pWZ9V45sk4d7peoE2njVprMRCBXrhJ5q5cYM7i6W+Uvw2qUrcfOSnuisrX7bEx3b3Wx4QA==", + "deprecated": "Package moved. Use @nats-io/transport-node from https://github.com/nats-io/nats.js", + "license": "Apache-2.0", + "dependencies": { + "nkeys.js": "1.1.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/nkeys.js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/nkeys.js/-/nkeys.js-1.1.0.tgz", + "integrity": "sha512-tB/a0shZL5UZWSwsoeyqfTszONTt4k2YS0tuQioMOD180+MbombYVgzDUYHlx+gejYK6rgf08n/2Df99WY0Sxg==", + "license": "Apache-2.0", + "dependencies": { + "tweetnacl": "1.0.3" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/tweetnacl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", + "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==", + "license": "Unlicense" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/modules/mesh-build-node/toolchain/package.json b/modules/mesh-build-node/toolchain/package.json new file mode 100644 index 00000000..42c5f21c --- /dev/null +++ b/modules/mesh-build-node/toolchain/package.json @@ -0,0 +1,15 @@ +{ + "name": "mesh-build-node-toolchain", + "version": "1.0.0", + "private": true, + "description": "The Node build module's toolchain: what npm ci installs into /toolchain/ (issue 1488).", + "dependencies": { + "@novox/mesh-sdk": "0.1.14", + "nats": "2.29.3" + }, + "devDependencies": { + "@types/node": "22.20.1", + "esbuild": "0.25.12", + "typescript": "5.9.3" + } +} diff --git a/modules/ssh-client/cmd/ssh-client-tools/client.go b/modules/ssh-client/cmd/ssh-client-tools/client.go index 454de9dd..3c2c5107 100644 --- a/modules/ssh-client/cmd/ssh-client-tools/client.go +++ b/modules/ssh-client/cmd/ssh-client-tools/client.go @@ -17,13 +17,15 @@ import ( "strings" "sync" "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // Client answers about one home's ~/.ssh. type Client struct { Home string UID int - Run Runner + Run forgekey.Runner Now func() time.Time // Settle is how long a path in ~/.ssh is left by the sweep after it changed (sweep.go): zero in tests. Settle time.Duration @@ -35,7 +37,7 @@ func NewClient() *Client { if home == "" { home, _ = os.UserHomeDir() } - return &Client{Home: home, UID: os.Getuid(), Run: ExecRunner, Now: time.Now, Settle: time.Minute} + return &Client{Home: home, UID: os.Getuid(), Run: forgekey.ExecRunner, Now: time.Now, Settle: time.Minute} } func (c *Client) ssh(name string) string { return filepath.Join(c.Home, ".ssh", name) } @@ -114,7 +116,7 @@ func matchesAny(host string, patterns []string) bool { return hit } -func named(what string, r Ran) error { +func named(what string, r forgekey.Ran) error { switch { case r.Err == "ENOENT": return fmt.Errorf("%s: the program is not installed on this machine", what) @@ -207,7 +209,7 @@ func (c *Client) Keys(ctx context.Context) ([]Key, error) { k.AgeDays = int(c.Now().Sub(info.ModTime()).Hours() / 24) } if pub, err := os.ReadFile(path + ".pub"); err == nil { - if pk, err := ParseKeyLine(string(pub)); err == nil { + if pk, err := forgekey.ParseKeyLine(string(pub)); err == nil { k.Type, k.Bits, k.Fingerprint, k.Comment, k.Weak = pk.Type, pk.Bits, pk.Fingerprint, pk.Comment, pk.Weak } } else { @@ -225,7 +227,7 @@ func (c *Client) Keys(ctx context.Context) ([]Key, error) { var derived string k.Passphrase, derived = c.passphrase(ctx, path) if derived != "" { - if pk, err := ParseKeyLine(derived); err == nil { + if pk, err := forgekey.ParseKeyLine(derived); err == nil { if k.Fingerprint != "" && !k.PublicMissing && pk.Fingerprint != k.Fingerprint { k.PublicMismatch = fmt.Sprintf("the key is %s; its .pub is %s", pk.Fingerprint, k.Fingerprint) } @@ -264,7 +266,7 @@ func (c *Client) passphrase(ctx context.Context, path string) (string, string) { // AuthorizedKey is one line of authorized_keys, by fingerprint. type AuthorizedKey struct { - PublicKey + forgekey.PublicKey Line int `json:"line"` InMesh bool `json:"in_mesh_region,omitempty"` } @@ -309,7 +311,7 @@ func (c *Client) readAuthorized() ([]AuthorizedKey, []int, error) { case t == "" || strings.HasPrefix(t, "#"): continue } - k, err := ParseKeyLine(t) + k, err := forgekey.ParseKeyLine(t) if err != nil { bad = append(bad, n+1) continue @@ -397,26 +399,26 @@ func atomicWrite(path string, data []byte, mode fs.FileMode) error { // ---- known_hosts ------------------------------------------------------------------------------- // knownFor is what known_hosts holds for one host (port 22, or [host]:port), by fingerprint. -func (c *Client) knownFor(ctx context.Context, host string, port int) ([]PublicKey, error) { +func (c *Client) knownFor(ctx context.Context, host string, port int) ([]forgekey.PublicKey, error) { name := host if port != 22 { name = fmt.Sprintf("[%s]:%d", host, port) } file := c.ssh("known_hosts") if _, err := os.Stat(file); os.IsNotExist(err) { - return []PublicKey{}, nil + return []forgekey.PublicKey{}, nil } r := c.Run(ctx, nil, "ssh-keygen", "-F", name, "-f", file) if r.Err != "" { return nil, named("ssh-keygen -F", r) } // Exit 1 with nothing printed is "not found". - keys := []PublicKey{} + keys := []forgekey.PublicKey{} for _, l := range strings.Split(r.Stdout, "\n") { if l = strings.TrimSpace(l); l == "" || strings.HasPrefix(l, "#") { continue } - if k, err := ParseKeyLine(l); err == nil { + if k, err := forgekey.ParseKeyLine(l); err == nil { k.Comment, k.Options = "", "" keys = append(keys, k) } @@ -425,17 +427,17 @@ func (c *Client) knownFor(ctx context.Context, host string, port int) ([]PublicK } // scan asks a host for its keys now. -func (c *Client) scan(ctx context.Context, host string, port int) ([]PublicKey, []string, error) { +func (c *Client) scan(ctx context.Context, host string, port int) ([]forgekey.PublicKey, []string, error) { r := c.Run(ctx, nil, "ssh-keyscan", "-T", "5", "-p", fmt.Sprint(port), host) if r.Err != "" { return nil, nil, named("ssh-keyscan", r) } - keys, lines := []PublicKey{}, []string{} + keys, lines := []forgekey.PublicKey{}, []string{} for _, l := range strings.Split(r.Stdout, "\n") { if l = strings.TrimSpace(l); l == "" || strings.HasPrefix(l, "#") { continue } - if k, err := ParseKeyLine(l); err == nil { + if k, err := forgekey.ParseKeyLine(l); err == nil { k.Comment, k.Options = "", "" keys = append(keys, k) lines = append(lines, l) @@ -455,7 +457,7 @@ func orElse(s, def string) string { } // compare says how what is known stands against what the host offers now. -func compare(known, live []PublicKey) string { +func compare(known, live []forgekey.PublicKey) string { if len(known) == 0 { return "not known: the first connection would ask" } diff --git a/modules/ssh-client/cmd/ssh-client-tools/firstline.go b/modules/ssh-client/cmd/ssh-client-tools/firstline.go new file mode 100644 index 00000000..e3aa9b47 --- /dev/null +++ b/modules/ssh-client/cmd/ssh-client-tools/firstline.go @@ -0,0 +1,13 @@ +package main + +import "strings" + +// firstLine is the first non-empty line of s, trimmed. +func firstLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + return l + } + } + return "" +} diff --git a/modules/ssh-client/cmd/ssh-client-tools/helpers_test.go b/modules/ssh-client/cmd/ssh-client-tools/helpers_test.go new file mode 100644 index 00000000..d2b180af --- /dev/null +++ b/modules/ssh-client/cmd/ssh-client-tools/helpers_test.go @@ -0,0 +1,21 @@ +package main + +import ( + "encoding/base64" + "encoding/binary" +) + +// ed25519Line is a public key line with a well-formed ed25519 blob. +func ed25519Line(seed byte, comment string) string { + blob := []byte{} + for _, part := range [][]byte{[]byte("ssh-ed25519"), make([]byte, 32)} { + part2 := append([]byte(nil), part...) + if len(part2) == 32 { + part2[0] = seed + } + l := make([]byte, 4) + binary.BigEndian.PutUint32(l, uint32(len(part2))) + blob = append(append(blob, l...), part2...) + } + return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(blob) + " " + comment +} diff --git a/modules/ssh-client/cmd/ssh-client-tools/keeper_test.go b/modules/ssh-client/cmd/ssh-client-tools/keeper_test.go new file mode 100644 index 00000000..0be30473 --- /dev/null +++ b/modules/ssh-client/cmd/ssh-client-tools/keeper_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 1480: the bundle's keeper is go/forgekey's, kind machine; no copy of it is left in this module. + +type recordState struct{ puts map[string][]byte } + +func (r *recordState) Put(key string, value any) (uint64, error) { + raw, _ := json.Marshal(value) + if r.puts == nil { + r.puts = map[string][]byte{} + } + r.puts[key] = raw + return uint64(len(r.puts)), nil +} + +// keygenStandIn writes the pair ssh-keygen -f names, as ssh-keygen would. +func keygenStandIn(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + var file, comment string + for i, a := range args { + switch a { + case "-f": + file = args[i+1] + case "-C": + comment = args[i+1] + } + } + if name != "ssh-keygen" || file == "" { + return forgekey.Ran{Status: 1, Stderr: "unexpected"} + } + os.WriteFile(file, []byte("PRIVATE\n"), 0o600) + os.WriteFile(file+".pub", []byte(ed25519Line(7, comment)+"\n"), 0o644) + return forgekey.Ran{} +} + +// 1480/AC1 +func TestTheBundlesKeeperIsForgekeysOfKindMachineWithItsEntryAtNodeAccount(t *testing.T) { + home, st := t.TempDir(), &recordState{} + k := keeperFor(home, "g14", "nox", keygenStandIn, st) + k.Now = func() time.Time { return time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC) } + if err := k.Once(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(home, ".ssh", "mesh_ed25519")); err != nil { + t.Errorf("the key file mesh_ed25519 was not made in an empty home: %v", err) + } + raw, ok := st.puts["g14.nox"] + if !ok || len(st.puts) != 1 { + t.Fatalf("entries = %v, want one at g14.nox", st.puts) + } + var e forgekey.KeyEntry + if err := json.Unmarshal(raw, &e); err != nil { + t.Fatal(err) + } + if e.Kind != "machine" || e.Node != "g14" || e.Account != "nox" || !strings.HasSuffix(e.Public, " nox@g14 (mesh)") { + t.Errorf("entry = %s", raw) + } +} + +// 1480/AC2 +func TestNoCopyOfTheKeeperIsLeftInThisModule(t *testing.T) { + copyOf := regexp.MustCompile(`(?m)^(type (KeyKeeper|KeyEntry|Runner|Ran)\b|func ParseKeyLine\b|func ExecRunner\b)`) + files, _ := filepath.Glob("*.go") + for _, f := range files { + raw, _ := os.ReadFile(f) + if m := copyOf.Find(raw); m != nil { + t.Errorf("%s declares %q, which go/forgekey holds", f, m) + } + } +} diff --git a/modules/ssh-client/cmd/ssh-client-tools/keykeeper.go b/modules/ssh-client/cmd/ssh-client-tools/keykeeper.go deleted file mode 100644 index be736d96..00000000 --- a/modules/ssh-client/cmd/ssh-client-tools/keykeeper.go +++ /dev/null @@ -1,115 +0,0 @@ -package main - -// The key this machine's account uses to reach the forge over SSH (novox/hq ADR 0399, ADR 0400; issue 955, -// area S1). It is made here, on the machine, once: ssh-keygen writes ~/.ssh/mesh_ed25519 only when it is -// missing, so the private half never leaves the machine and an existing key is never replaced. The public -// half and its fingerprint go into this module's `keys` state, one entry per account under ., -// for the forge's provider to register. The tools bundle is the one process of this module, so the keeper -// runs inside it beside the tools (main.go), the way claude-code reports what it holds. - -import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - "time" -) - -// KeyFile is the key's name under ~/.ssh: never one of the operator's own id_* files. -const KeyFile = "mesh_ed25519" - -// KeyKind is the kind of key this process makes: the machine's account's own (ADR 0400 §1). -const KeyKind = "machine" - -// KeyRetry is how soon it looks again after a failure. -const KeyRetry = 30 * time.Second - -// KeyRound is how often the keeper looks again. -const KeyRound = 10 * time.Minute - -// KeyEntry is one value of the keys state. It carries the public half only. -type KeyEntry struct { - Node string `json:"node"` - Account string `json:"account"` - Kind string `json:"kind"` - Public string `json:"public"` - Fingerprint string `json:"fingerprint"` - At string `json:"at"` -} - -// KeyPutter is the keys state, as far as the keeper needs it. -type KeyPutter interface { - Put(key string, value any) error -} - -// KeyKeeper keeps one account's key and its entry. -type KeyKeeper struct { - Home, Node, Account string - Run Runner - State KeyPutter - Now func() time.Time - - reported string // the public half and fingerprint last put -} - -func (k *KeyKeeper) path() string { return filepath.Join(k.Home, ".ssh", KeyFile) } - -// Once makes the key when it is missing and puts the entry when it differs from the one last put. -func (k *KeyKeeper) Once(ctx context.Context) error { - if err := k.ensure(ctx); err != nil { - return err - } - raw, err := os.ReadFile(k.path() + ".pub") - if err != nil { - return fmt.Errorf("the public half: %w", err) - } - pk, err := ParseKeyLine(string(raw)) - if err != nil { - return fmt.Errorf("%s.pub: %w", k.path(), err) - } - public := strings.TrimSpace(string(raw)) - if sig := public + "|" + pk.Fingerprint; sig == k.reported { - return nil - } - entry := KeyEntry{Node: k.Node, Account: k.Account, Kind: KeyKind, Public: public, - Fingerprint: pk.Fingerprint, At: k.Now().UTC().Format(time.RFC3339)} - if err := k.State.Put(k.Node+"."+k.Account, entry); err != nil { - return err - } - k.reported = public + "|" + pk.Fingerprint - return nil -} - -// ensure makes the pair when the private half is missing. Never over an existing key. -func (k *KeyKeeper) ensure(ctx context.Context) error { - if _, err := os.Lstat(k.path()); err == nil { - return nil - } else if !os.IsNotExist(err) { - return err - } - if err := os.MkdirAll(filepath.Dir(k.path()), 0o700); err != nil { - return err - } - r := k.Run(ctx, nil, "ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", k.Account+"@"+k.Node+" (mesh)", "-f", k.path()) - if r.Status != 0 || r.Err != "" { - return fmt.Errorf("ssh-keygen: %s", firstLine(r.Err+" "+r.Stderr)) - } - return nil -} - -// Keep runs Once now and every KeyRound, reporting a failure through say and trying again at the next round. -func (k *KeyKeeper) Keep(ctx context.Context, say func(string, ...any)) { - for { - pause := KeyRound - if err := k.Once(ctx); err != nil { - say("the forge key: %v", err) - pause = KeyRetry // the state or the bus's grant may arrive after the process - } - select { - case <-ctx.Done(): - return - case <-time.After(pause): - } - } -} diff --git a/modules/ssh-client/cmd/ssh-client-tools/keykeeper_bus_test.go b/modules/ssh-client/cmd/ssh-client-tools/keykeeper_bus_test.go index 80df4fb5..cea182b5 100644 --- a/modules/ssh-client/cmd/ssh-client-tools/keykeeper_bus_test.go +++ b/modules/ssh-client/cmd/ssh-client-tools/keykeeper_bus_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/forgekey" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -170,7 +171,7 @@ func TestTheKeyProcessMakesTheKeyAndPutsItsEntryOverTheBus(t *testing.T) { if raw == nil { t.Fatalf("no entry %s in the keys state within 60 s", key) } - var e KeyEntry + var e forgekey.KeyEntry if err := json.Unmarshal(raw, &e); err != nil { t.Fatal(err) } @@ -180,7 +181,7 @@ func TestTheKeyProcessMakesTheKeyAndPutsItsEntryOverTheBus(t *testing.T) { if strings.Contains(strings.ToUpper(string(raw)), "PRIVATE") { t.Errorf("the entry carries private material: %s", raw) } - info, err := os.Stat(filepath.Join(home, ".ssh", KeyFile)) + info, err := os.Stat(filepath.Join(home, ".ssh", forgekey.KeyFile)) if err != nil { t.Fatal(err) } diff --git a/modules/ssh-client/cmd/ssh-client-tools/keykeeper_test.go b/modules/ssh-client/cmd/ssh-client-tools/keykeeper_test.go deleted file mode 100644 index 0ceb52e1..00000000 --- a/modules/ssh-client/cmd/ssh-client-tools/keykeeper_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package main - -import ( - "context" - "encoding/base64" - "encoding/binary" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -// Issue 955, area S1: the key the machine's account makes for the forge, and its entry in the keys state. - -type fakeState struct{ puts map[string][]byte } - -func (f *fakeState) Put(key string, value any) error { - raw, _ := json.Marshal(value) - if f.puts == nil { - f.puts = map[string][]byte{} - } - f.puts[key] = raw - return nil -} - -// ed25519Line is a public key line with a well-formed ed25519 blob. -func ed25519Line(seed byte, comment string) string { - blob := []byte{} - for _, part := range [][]byte{[]byte("ssh-ed25519"), make([]byte, 32)} { - part2 := append([]byte(nil), part...) - if len(part2) == 32 { - part2[0] = seed - } - l := make([]byte, 4) - binary.BigEndian.PutUint32(l, uint32(len(part2))) - blob = append(append(blob, l...), part2...) - } - return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(blob) + " " + comment -} - -// keygenRunner stands in for ssh-keygen -t ed25519: it writes the pair the command names. -func keygenRunner(calls *[][]string) Runner { - return func(ctx context.Context, env []string, name string, args ...string) Ran { - *calls = append(*calls, append([]string{name}, args...)) - var file, comment string - for i, a := range args { - switch a { - case "-f": - file = args[i+1] - case "-C": - comment = args[i+1] - } - } - if name != "ssh-keygen" || file == "" { - return Ran{Status: 1, Stderr: "unexpected"} - } - os.WriteFile(file, []byte("-----BEGIN OPENSSH PRIVATE KEY-----\nPRIVATE-SECRET\n"), 0o600) - os.WriteFile(file+".pub", []byte(ed25519Line(7, comment)+"\n"), 0o644) - return Ran{} - } -} - -func newKeeper(t *testing.T, home string, st *fakeState, calls *[][]string) *KeyKeeper { - return &KeyKeeper{Home: home, Node: "g14", Account: "nox", Run: keygenRunner(calls), State: st, - Now: func() time.Time { return time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) }} -} - -func TestKeeperMakesTheKeyAndWritesItsEntry(t *testing.T) { - home := t.TempDir() - st, calls := &fakeState{}, &[][]string{} - if err := newKeeper(t, home, st, calls).Once(context.Background()); err != nil { - t.Fatal(err) - } - raw, ok := st.puts["g14.nox"] - if !ok || len(st.puts) != 1 { - t.Fatalf("entries = %v, want one under g14.nox", st.puts) - } - var e map[string]string - json.Unmarshal(raw, &e) - if e["node"] != "g14" || e["account"] != "nox" || e["kind"] != "machine" || e["at"] != "2026-10-16T12:00:00Z" { - t.Errorf("entry = %v", e) - } - if !strings.HasPrefix(e["public"], "ssh-ed25519 ") || !strings.HasSuffix(e["public"], " nox@g14 (mesh)") { - t.Errorf("public = %q", e["public"]) - } - if !strings.HasPrefix(e["fingerprint"], "SHA256:") { - t.Errorf("fingerprint = %q", e["fingerprint"]) - } - if strings.Contains(strings.ToUpper(string(raw)), "PRIVATE") { - t.Errorf("the entry carries private material: %s", raw) - } - cmd := strings.Join((*calls)[0], " ") - for _, want := range []string{"-t ed25519", "-N ", "-C nox@g14 (mesh)", filepath.Join(home, ".ssh", "mesh_ed25519")} { - if !strings.Contains(cmd, want) { - t.Errorf("ssh-keygen call %q lacks %q", cmd, want) - } - } -} - -func TestKeeperKeepsAnExistingKeyByteForByte(t *testing.T) { - home := t.TempDir() - os.MkdirAll(filepath.Join(home, ".ssh"), 0o700) - priv, pub := filepath.Join(home, ".ssh", "mesh_ed25519"), filepath.Join(home, ".ssh", "mesh_ed25519.pub") - os.WriteFile(priv, []byte("mine"), 0o600) - os.WriteFile(pub, []byte(ed25519Line(9, "nox@g14 (mesh)")+"\n"), 0o644) - st, calls := &fakeState{}, &[][]string{} - k := newKeeper(t, home, st, calls) - if err := k.Once(context.Background()); err != nil { - t.Fatal(err) - } - if len(*calls) != 0 { - t.Errorf("ssh-keygen ran over an existing key: %v", *calls) - } - if b, _ := os.ReadFile(priv); string(b) != "mine" { - t.Errorf("private file changed: %q", b) - } - if _, ok := st.puts["g14.nox"]; !ok { - t.Errorf("no entry for the existing key") - } - // A second round with nothing changed writes nothing. - delete(st.puts, "g14.nox") - k.Once(context.Background()) - if len(st.puts) != 0 { - t.Errorf("an unchanged key was put again") - } -} - -func TestKeeperNeverTouchesTheOperatorsOwnKeys(t *testing.T) { - home := t.TempDir() - os.MkdirAll(filepath.Join(home, ".ssh"), 0o700) - own := filepath.Join(home, ".ssh", "id_ed25519") - os.WriteFile(own, []byte("operator"), 0o600) - st, calls := &fakeState{}, &[][]string{} - newKeeper(t, home, st, calls).Once(context.Background()) - if b, _ := os.ReadFile(own); string(b) != "operator" { - t.Errorf("the operator's key changed") - } -} diff --git a/modules/ssh-client/cmd/ssh-client-tools/keys.go b/modules/ssh-client/cmd/ssh-client-tools/keys.go deleted file mode 100644 index b248badd..00000000 --- a/modules/ssh-client/cmd/ssh-client-tools/keys.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -// Keys, by their public half only. A fingerprint is computed here from the public key's bytes, as -// ssh-keygen -l does (SHA256 of the key blob, unpadded base64); a private key is never read by this -// process — its first line, the PEM header, says it is one, and ssh-keygen is asked about it. - -import ( - "crypto/sha256" - "encoding/base64" - "encoding/binary" - "fmt" - "math/big" - "strings" -) - -// KeyTypes are the public key algorithms an authorized_keys or known_hosts line can carry. -var KeyTypes = map[string]bool{ - "ssh-ed25519": true, "ssh-rsa": true, "ssh-dss": true, - "ecdsa-sha2-nistp256": true, "ecdsa-sha2-nistp384": true, "ecdsa-sha2-nistp521": true, - "sk-ssh-ed25519@openssh.com": true, "sk-ecdsa-sha2-nistp256@openssh.com": true, -} - -// PublicKey is one public key as a line carries it. -type PublicKey struct { - Type string `json:"type"` - Bits int `json:"bits"` - Fingerprint string `json:"fingerprint"` - Comment string `json:"comment"` - Options string `json:"options,omitempty"` - Weak string `json:"weak,omitempty"` -} - -// ParseKeyLine reads one authorized_keys line (options, type, key, comment) or a public key file's. -func ParseKeyLine(line string) (PublicKey, error) { - fields := fieldsQuoted(strings.TrimSpace(line)) - for i, f := range fields { - if !KeyTypes[f] || i+1 >= len(fields) { - continue - } - k := PublicKey{Type: f, Options: strings.Join(fields[:i], " "), Comment: strings.Join(fields[i+2:], " ")} - blob, err := base64.StdEncoding.DecodeString(fields[i+1]) - if err != nil { - return k, fmt.Errorf("the key after %s is not base64", f) - } - sum := sha256.Sum256(blob) - k.Fingerprint = "SHA256:" + base64.RawStdEncoding.EncodeToString(sum[:]) - k.Bits = bitsOf(f, blob) - switch { - case f == "ssh-dss": - k.Weak = "DSA: refused by current OpenSSH" - case f == "ssh-rsa" && k.Bits > 0 && k.Bits < 3072: - k.Weak = fmt.Sprintf("RSA of %d bits: below 3072", k.Bits) - } - return k, nil - } - return PublicKey{}, fmt.Errorf("no public key on this line") -} - -// fieldsQuoted splits on spaces outside double quotes, as sshd reads an options field. -func fieldsQuoted(s string) []string { - var out []string - var cur strings.Builder - quoted := false - for _, ch := range s { - switch { - case ch == '"': - quoted = !quoted - cur.WriteRune(ch) - case (ch == ' ' || ch == '\t') && !quoted: - if cur.Len() > 0 { - out = append(out, cur.String()) - cur.Reset() - } - default: - cur.WriteRune(ch) - } - } - if cur.Len() > 0 { - out = append(out, cur.String()) - } - return out -} - -// bitsOf reads a key's size from its blob: the RSA modulus, the curve, or ed25519's fixed 256. -func bitsOf(kind string, blob []byte) int { - strs := [][]byte{} - for len(blob) >= 4 { - n := binary.BigEndian.Uint32(blob) - if int(n) > len(blob)-4 { - break - } - strs = append(strs, blob[4:4+n]) - blob = blob[4+n:] - } - switch { - case strings.Contains(kind, "ed25519"): - return 256 - case kind == "ssh-rsa" && len(strs) >= 3: - return new(big.Int).SetBytes(strs[2]).BitLen() - case kind == "ssh-dss" && len(strs) >= 2: - return new(big.Int).SetBytes(strs[1]).BitLen() - case strings.Contains(kind, "nistp256"): - return 256 - case strings.Contains(kind, "nistp384"): - return 384 - case strings.Contains(kind, "nistp521"): - return 521 - } - return 0 -} diff --git a/modules/ssh-client/cmd/ssh-client-tools/main.go b/modules/ssh-client/cmd/ssh-client-tools/main.go index 98ee3e82..d93b5493 100644 --- a/modules/ssh-client/cmd/ssh-client-tools/main.go +++ b/modules/ssh-client/cmd/ssh-client-tools/main.go @@ -15,6 +15,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/forgekey" ) func main() { @@ -141,7 +142,7 @@ func flag(args map[string]any, key string, def bool) bool { return def } -// startKeeper runs the forge key's keeper (keykeeper.go) when the runtime launched this process for a node: +// startKeeper runs the forge key's keeper (go/forgekey) when the runtime launched this process for a node: // outside a launch — a build, a check — there is no state to put the entry in. func startKeeper() { node := strings.TrimSpace(os.Getenv("MESH_NODE")) @@ -151,22 +152,27 @@ func startKeeper() { return } go keepSwept(c) - k := &KeyKeeper{Home: c.Home, Node: node, Account: u.Username, Run: ExecRunner, State: keysState{}, Now: time.Now} + k := keeperFor(c.Home, node, u.Username, forgekey.ExecRunner, keysState{}) k.Keep(context.Background(), func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[ssh-client] "+f+"\n", a...) }) } -// keepSwept sweeps ~/.ssh (sweep.go) now and every KeyRound. +// keeperFor is the machine's account's keeper: go/forgekey's, of kind machine (ADR 0400 §1, ADR 0466 §2), whose key +// file is ~/.ssh/mesh_ed25519 and whose entry lies at . in the keys state. +func keeperFor(home, node, account string, run forgekey.Runner, state forgekey.KeyPutter) *forgekey.KeyKeeper { + return &forgekey.KeyKeeper{Home: home, Node: node, Account: account, Kind: forgekey.KindMachine, Run: run, State: state, Now: time.Now} +} + +// keepSwept sweeps ~/.ssh (sweep.go) now and every forgekey.KeyRound. func keepSwept(c *Client) { say := func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[ssh-client] "+f+"\n", a...) } for { c.SweepOnce(time.Now(), say) - time.Sleep(KeyRound) + time.Sleep(forgekey.KeyRound) } } type keysState struct{} -func (keysState) Put(key string, value any) error { - _, err := stdio.State("keys").Put(key, value) - return err +func (keysState) Put(key string, value any) (uint64, error) { + return stdio.State("keys").Put(key, value) } diff --git a/modules/ssh-client/cmd/ssh-client-tools/runner.go b/modules/ssh-client/cmd/ssh-client-tools/runner.go deleted file mode 100644 index 35ae5798..00000000 --- a/modules/ssh-client/cmd/ssh-client-tools/runner.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import ( - "bytes" - "context" - "errors" - "fmt" - "os" - "os/exec" - "strings" - "time" -) - -// Ran is what a command did: its output, its exit status, and why it never ran to an answer. -type Ran struct { - Stdout string - Stderr string - Status int - // Err is "ENOENT" when the program is not installed, or says it was ended for taking too long. - Err string -} - -// Runner runs one command with extra environment words, so every tool can be tested without ssh. -type Runner func(ctx context.Context, env []string, name string, args ...string) Ran - -// CallTimeout bounds one command: below the runtime's thirty-second call limit. -const CallTimeout = 20 * time.Second - -// ExecRunner runs a command on this machine, with no terminal and nothing on its stdin, bounded by -// CallTimeout. -func ExecRunner(ctx context.Context, env []string, name string, args ...string) Ran { - ctx, cancel := context.WithTimeout(ctx, CallTimeout) - defer cancel() - cmd := exec.CommandContext(ctx, name, args...) - cmd.Env = append(os.Environ(), env...) - var out, errb bytes.Buffer - cmd.Stdout, cmd.Stderr = &out, &errb - err := cmd.Run() - r := Ran{Stdout: out.String(), Stderr: errb.String()} - var exitErr *exec.ExitError - switch { - case errors.Is(ctx.Err(), context.DeadlineExceeded): - r.Status, r.Err = 124, fmt.Sprintf("no answer within %d s", int(CallTimeout/time.Second)) - case errors.Is(err, exec.ErrNotFound): - r.Status, r.Err = 127, "ENOENT" - case errors.As(err, &exitErr): - r.Status = exitErr.ExitCode() - case err != nil: - r.Status, r.Err = 1, err.Error() - } - return r -} - -func firstLine(s string) string { - for _, l := range strings.Split(s, "\n") { - if l = strings.TrimSpace(l); l != "" { - return l - } - } - return "" -} diff --git a/modules/ssh-client/cmd/ssh-client-tools/ssh_test.go b/modules/ssh-client/cmd/ssh-client-tools/ssh_test.go index 763540cb..3428e042 100644 --- a/modules/ssh-client/cmd/ssh-client-tools/ssh_test.go +++ b/modules/ssh-client/cmd/ssh-client-tools/ssh_test.go @@ -12,6 +12,8 @@ import ( "sync" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // Public keys made for these tests only; their private halves were never kept. @@ -29,7 +31,7 @@ type call struct { } type fake struct { - answer func(c call) Ran + answer func(c call) forgekey.Ran mu sync.Mutex calls []call } @@ -41,13 +43,13 @@ func (f *fake) recorded() []call { return append([]call(nil), f.calls...) } -func (f *fake) run(_ context.Context, env []string, name string, args ...string) Ran { +func (f *fake) run(_ context.Context, env []string, name string, args ...string) forgekey.Ran { c := call{env, name, args} f.mu.Lock() f.calls = append(f.calls, c) f.mu.Unlock() if f.answer == nil { - return Ran{Status: 1} + return forgekey.Ran{Status: 1} } return f.answer(c) } @@ -130,19 +132,19 @@ func TestKeysAreDescribedByTheirPublicHalfAndAPassphraseIsAskedNotRead(t *testin files := map[string]string{"config": layout["config"], "id_ed25519": "-----BEGIN OPENSSH PRIVATE KEY-----\nnot a key\n", "id_ed25519.pub": edPub + "\n", "id_box": "-----BEGIN OPENSSH PRIVATE KEY-----\nnot a key\n", "id_box.pub": rsaPub + "\n", "stray": "-----BEGIN RSA PRIVATE KEY-----\nnot a key\n", "notes.txt": "hello\n"} - f := &fake{answer: func(c call) Ran { + f := &fake{answer: func(c call) forgekey.Ran { if c.name == "ssh-keygen" && c.args[0] == "-y" { if strings.HasSuffix(c.args[len(c.args)-1], "id_box") { - return Ran{Status: 1, Stderr: "Load key \"id_box\": incorrect passphrase supplied to decrypt private key\n"} + return forgekey.Ran{Status: 1, Stderr: "Load key \"id_box\": incorrect passphrase supplied to decrypt private key\n"} } if strings.HasSuffix(c.args[len(c.args)-1], "id_ed25519") { - return Ran{Stdout: edPub + "\n"} + return forgekey.Ran{Stdout: edPub + "\n"} } } if c.name == "ssh-keygen" && c.args[0] == "-l" { - return Ran{Stdout: "256 " + edFP + " no comment (ED25519)\n"} + return forgekey.Ran{Stdout: "256 " + edFP + " no comment (ED25519)\n"} } - return Ran{Status: 1, Stderr: "unexpected"} + return forgekey.Ran{Status: 1, Stderr: "unexpected"} }} keys, err := client(home(t, files), f).Keys(context.Background()) if err != nil { @@ -173,7 +175,7 @@ func TestKeysAreDescribedByTheirPublicHalfAndAPassphraseIsAskedNotRead(t *testin func TestAPublicHalfThatIsAnotherKeysIsFound(t *testing.T) { files := map[string]string{"config": layout["config"], "id_ed25519": "-----BEGIN OPENSSH PRIVATE KEY-----\n", "id_ed25519.pub": rsaPub + "\n"} - f := &fake{answer: func(c call) Ran { return Ran{Stdout: edPub + "\n"} }} + f := &fake{answer: func(c call) forgekey.Ran { return forgekey.Ran{Stdout: edPub + "\n"} }} keys, _ := client(home(t, files), f).Keys(context.Background()) if len(keys) != 1 || keys[0].PublicMismatch == "" || keys[0].Fingerprint != edFP { t.Fatalf("%+v", keys) @@ -181,15 +183,15 @@ func TestAPublicHalfThatIsAnotherKeysIsFound(t *testing.T) { } func TestFingerprintsAreSshKeygens(t *testing.T) { - k, err := ParseKeyLine("from=\"10.0.0.0/8,192.0.2.1\",no-pty " + edPub) + k, err := forgekey.ParseKeyLine("from=\"10.0.0.0/8,192.0.2.1\",no-pty " + edPub) if err != nil || k.Fingerprint != edFP || k.Bits != 256 || k.Comment != "op@laptop" || !strings.HasPrefix(k.Options, "from=") { t.Fatalf("%+v %v", k, err) } - k, _ = ParseKeyLine(rsaPub) + k, _ = forgekey.ParseKeyLine(rsaPub) if k.Fingerprint != rsaFP || k.Bits != 2048 || !strings.Contains(k.Weak, "below 3072") { t.Fatalf("%+v", k) } - if _, err := ParseKeyLine("ssh-ed25519 !!!notbase64"); err == nil { + if _, err := forgekey.ParseKeyLine("ssh-ed25519 !!!notbase64"); err == nil { t.Fatal("garbage accepted") } } @@ -245,16 +247,16 @@ func scanOf(host, pub string) string { return host + " " + pub + "\n" } func TestKnownHostComparesWhatIsKnownWithWhatIsOffered(t *testing.T) { h := home(t, map[string]string{"config": layout["config"], "known_hosts": "ace.internal " + edPub + "\n"}) offered := edPub - f := &fake{answer: func(c call) Ran { + f := &fake{answer: func(c call) forgekey.Ran { switch { case c.name == "ssh-keygen" && c.args[0] == "-F": - return Ran{Stdout: "# Host ace.internal found: line 1\nace.internal " + edPub + "\n"} + return forgekey.Ran{Stdout: "# Host ace.internal found: line 1\nace.internal " + edPub + "\n"} case c.name == "ssh-keyscan": - return Ran{Stdout: scanOf("ace.internal", offered)} + return forgekey.Ran{Stdout: scanOf("ace.internal", offered)} case c.name == "ssh-keygen" && c.args[0] == "-R": - return Ran{} + return forgekey.Ran{} } - return Ran{Status: 1} + return forgekey.Ran{Status: 1} }} c := client(h, f) got, err := c.KnownHost(context.Background(), "ace.internal", 22, false) @@ -293,11 +295,11 @@ func TestKnownHostComparesWhatIsKnownWithWhatIsOffered(t *testing.T) { func TestAnUnreachableHostIsNotRefreshed(t *testing.T) { h := home(t, map[string]string{"known_hosts": ""}) - f := &fake{answer: func(c call) Ran { + f := &fake{answer: func(c call) forgekey.Ran { if c.name == "ssh-keyscan" { - return Ran{Status: 1} + return forgekey.Ran{Status: 1} } - return Ran{Status: 1} + return forgekey.Ran{Status: 1} }} got, err := client(h, f).KnownHost(context.Background(), "gone.example", 22, false) if err != nil || !strings.HasPrefix(got["state"].(string), "unreachable") { @@ -310,7 +312,7 @@ func TestAnUnreachableHostIsNotRefreshed(t *testing.T) { func TestTestSaysHowItAuthenticatedOrWhyNot(t *testing.T) { ok := "debug1: Connecting to ace.internal [10.0.0.2] port 22.\ndebug1: Server accepts key: /h/.ssh/id_ed25519 ED25519 " + edFP + "\nAuthenticated to ace.internal ([10.0.0.2]:22) using \"publickey\".\n" - f := &fake{answer: func(c call) Ran { return Ran{Stderr: ok} }} + f := &fake{answer: func(c call) forgekey.Ran { return forgekey.Ran{Stderr: ok} }} got, err := client(t.TempDir(), f).Test(context.Background(), "ace") if err != nil || got["ok"] != true || got["method"] != "publickey" || got["connected_to"] != "10.0.0.2:22" { t.Fatalf("%v %v", got, err) @@ -320,7 +322,7 @@ func TestTestSaysHowItAuthenticatedOrWhyNot(t *testing.T) { t.Fatalf("not a batch test: %s", args) } denied := "debug1: Offering public key: /h/.ssh/id_box RSA " + rsaFP + "\nop@ace.internal: Permission denied (publickey).\n" - f = &fake{answer: func(c call) Ran { return Ran{Status: 255, Stderr: denied} }} + f = &fake{answer: func(c call) forgekey.Ran { return forgekey.Ran{Status: 255, Stderr: denied} }} got, _ = client(t.TempDir(), f).Test(context.Background(), "ace") if got["ok"] != false || got["why"] != "no key it offered was accepted" || !reflect.DeepEqual(got["offered"], []string{"/h/.ssh/id_box"}) { t.Fatalf("%v", got) @@ -337,14 +339,14 @@ func TestCheckFindsWhatIsWrongAndSaysWhatItDidNotCheck(t *testing.T) { h := home(t, files) os.Chmod(filepath.Join(h, ".ssh", "config"), 0o666) os.Chmod(filepath.Join(h, ".ssh", "id_ed25519"), 0o644) - f := &fake{answer: func(c call) Ran { + f := &fake{answer: func(c call) forgekey.Ran { switch { case c.name == "ssh-keygen" && c.args[0] == "-y": - return Ran{Stdout: edPub} + return forgekey.Ran{Stdout: edPub} case c.name == "ssh-keyscan": - return Ran{Stdout: scanOf("x", edPub)} + return forgekey.Ran{Stdout: scanOf("x", edPub)} } - return Ran{Status: 1} + return forgekey.Ran{Status: 1} }} got, err := client(h, f).Check(context.Background(), true) if err != nil { @@ -381,7 +383,7 @@ func TestCheckFindsWhatIsWrongAndSaysWhatItDidNotCheck(t *testing.T) { // as problems, and says when no sweep has run. func TestCheckShowsAFailedSweep(t *testing.T) { h := home(t, map[string]string{"config": layout["config"], "known_hosts": ""}) - f := &fake{answer: func(call) Ran { return Ran{Status: 1} }} + f := &fake{answer: func(call) forgekey.Ran { return forgekey.Ran{Status: 1} }} c := client(h, f) got, _ := c.Check(context.Background(), false) if !slices.ContainsFunc(got["not_checked"].([]string), func(s string) bool { return strings.Contains(s, "the sweep") }) { diff --git a/modules/ssh-client/cmd/ssh-client-tools/sweep.go b/modules/ssh-client/cmd/ssh-client-tools/sweep.go index c297bbf5..b22ea2de 100644 --- a/modules/ssh-client/cmd/ssh-client-tools/sweep.go +++ b/modules/ssh-client/cmd/ssh-client-tools/sweep.go @@ -7,7 +7,7 @@ package main // can lock the operator out); a folder holding a key is gone into rather than moved whole. A socket // is ssh's own runtime and stays, and so do the files ssh and sshd read by themselves (SSHReads). A // path changed in the last Settle is being written (a temporary file of atomicWrite or ssh-keygen) and -// waits for the next sweep. The process sweeps when it starts and every KeyRound (main.go); what the +// waits for the next sweep. The process sweeps when it starts and every KeyRound of go/forgekey (main.go); what the // last sweep did, or why it failed, is kept beside the retired folder for ssh_client_check (issue 1020). import ( @@ -21,19 +21,21 @@ import ( "strings" "syscall" "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // Declared are the paths under ~/.ssh this module declares (module.json) or its own process makes // (keykeeper.go). A test checks the manifest against this list. var Declared = map[string]bool{ - "config": true, - "known_hosts": true, - "config.d": true, - "config.d/00-mesh": true, - "config.d/10-forge": true, - KeyFile: true, - KeyFile + ".pub": true, - "authorized_keys": true, + "config": true, + "known_hosts": true, + "config.d": true, + "config.d/00-mesh": true, + "config.d/10-forge": true, + forgekey.KeyFile: true, + forgekey.KeyFile + ".pub": true, + "authorized_keys": true, } // SSHReads are the files ssh and sshd read in ~/.ssh by themselves, though this module writes none of @@ -371,7 +373,7 @@ func publicKey(path string) bool { return false } line, _, _ := strings.Cut(string(raw), "\n") - if _, err := ParseKeyLine(line); err == nil { + if _, err := forgekey.ParseKeyLine(line); err == nil { return true } f := strings.Fields(line) diff --git a/modules/ssh-client/cmd/ssh-client-tools/sweep_test.go b/modules/ssh-client/cmd/ssh-client-tools/sweep_test.go index ede72c84..cf60a604 100644 --- a/modules/ssh-client/cmd/ssh-client-tools/sweep_test.go +++ b/modules/ssh-client/cmd/ssh-client-tools/sweep_test.go @@ -12,6 +12,8 @@ import ( "syscall" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // ADR 0415, issue 1017: the sweep keeps ~/.ssh to what this module knows and moves the rest out. @@ -73,7 +75,7 @@ func TestSweepKeepsWhatItKnows(t *testing.T) { t.Fatal(err) } for _, rel := range []string{"config", "known_hosts", "authorized_keys", "config.d/00-mesh", "config.d/10-forge", - KeyFile, KeyFile + ".pub", "id_ed25519", "id_ed25519.pub", "work_key", "work_key.pub", "work_key-cert.pub", "agent_only.pub", "removed-drone-20261004/id_drone", "authorized_keys2", "id_token"} { + forgekey.KeyFile, forgekey.KeyFile + ".pub", "id_ed25519", "id_ed25519.pub", "work_key", "work_key.pub", "work_key-cert.pub", "agent_only.pub", "removed-drone-20261004/id_drone", "authorized_keys2", "id_token"} { if !exists(filepath.Join(dir, rel)) { t.Errorf("%s was moved; it is known", rel) } diff --git a/modules/ssh-client/go.mod b/modules/ssh-client/go.mod index 9be3c232..77590b40 100644 --- a/modules/ssh-client/go.mod +++ b/modules/ssh-client/go.mod @@ -1,22 +1,22 @@ module sshclient -go 1.25.0 +go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d github.com/nats-io/nats-server/v2 v2.11.17 - github.com/nats-io/nats.go v1.51.0 + github.com/nats-io/nats.go v1.54.0 ) require ( github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.20.0 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/sys v0.43.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/ssh-client/go.sum b/modules/ssh-client/go.sum index f84463ba..a1de2261 100644 --- a/modules/ssh-client/go.sum +++ b/modules/ssh-client/go.sum @@ -1,11 +1,19 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020200715-9e296295c05e h1:uf2o2A6dBOrISDUVw6U7+fv8ylwwF+UfRrGWKIQMFOE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020200715-9e296295c05e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020203945-1572e50e0ef0 h1:mr9HyARwzeZT7FxFHv1Xc2z7QErvDZ/F/pBMIpjGSd0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020203945-1572e50e0ef0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d h1:mmOCJphjQN15jNVh+sMQqnmJ3UiYltiuQRg/zw3wFtw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= @@ -14,14 +22,22 @@ github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnY github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=