novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Merge pull request 'The Go and the Node build modules, ssh-client on the SDK's key keeper, and gitea's read-only builder accounts (design 47 R4c-3)' (#389) from feat/r4c3-build-modules into main

Browse Source
This commit was merged in pull request #389.
This commit is contained in:
mesh-tools
2026-10-20 21:43:14 +00:00
parent 06d5dc9830 bcee61edf7
commit 199f72fed2
130 changed files with 10043 additions and 582 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/gitea/cmd/forge-account-provider/builder_test.go
+356
View File
@@ -0,0 +1,356 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/provider"
"gitea/internal/forgehttp"
)
// Issue 1481 (novox/hq ADR 0466 §3): the build modules' keys on one forge user that reads and pushes nothing. The forge
// is faked with the teams, members and keys it keeps, so a team's shape and a key's right can be read back.
type forgeKey struct {
ID int64
Title string
Key string
ReadOnly bool
}
type teamForge struct {
mu sync.Mutex
users map[string]bool
teams map[string]map[string]any // by name: the shape last sent
ids map[string]int64
members map[string][]string // by team name
keys map[string][]forgeKey
next int64
}
func newTeamForge() *teamForge {
return &teamForge{users: map[string]bool{}, teams: map[string]map[string]any{}, ids: map[string]int64{},
members: map[string][]string{}, keys: map[string][]forgeKey{}, next: 10}
}
var (
teamRe = regexp.MustCompile(`^/api/v1/teams/(\d+)(?:/members/([^/]+))?$`)
userKeyRe = regexp.MustCompile(`^/api/v1/(?:admin/)?users/([^/]+)/keys(?:/(\d+))?$`)
)
func (f *teamForge) teamByID(id string) string {
for name, n := range f.ids {
if fmt.Sprint(n) == id {
return name
}
}
return ""
}
func (f *teamForge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
defer f.mu.Unlock()
body, _ := io.ReadAll(r.Body)
var in map[string]any
_ = json.Unmarshal(body, &in)
reply := func(code int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
if v != nil {
_ = json.NewEncoder(w).Encode(v)
}
}
p := r.URL.Path
if m := userKeyRe.FindStringSubmatch(p); m != nil {
switch r.Method {
case http.MethodGet:
out := []map[string]any{}
for _, k := range f.keys[m[1]] {
out = append(out, map[string]any{"id": k.ID, "title": k.Title, "key": k.Key})
}
reply(200, out)
case http.MethodPost:
f.next++
ro, _ := in["read_only"].(bool)
f.keys[m[1]] = append(f.keys[m[1]], forgeKey{ID: f.next, Title: in["title"].(string), Key: in["key"].(string), ReadOnly: ro})
reply(201, map[string]any{"id": f.next})
case http.MethodDelete:
var kept []forgeKey
for _, k := range f.keys[m[1]] {
if fmt.Sprint(k.ID) != m[2] {
kept = append(kept, k)
}
}
f.keys[m[1]] = kept
reply(204, nil)
}
return
}
if m := teamRe.FindStringSubmatch(p); m != nil {
name := f.teamByID(m[1])
switch {
case name == "":
reply(404, map[string]any{"message": "no team"})
case r.Method == http.MethodPatch && m[2] == "":
f.teams[name] = in
reply(200, map[string]any{})
case r.Method == http.MethodPut:
for _, u := range f.members[name] {
if u == m[2] {
reply(204, nil)
return
}
}
f.members[name] = append(f.members[name], m[2])
reply(204, nil)
default:
http.Error(w, "unexpected "+r.Method+" "+p, 400)
}
return
}
switch {
case r.Method == http.MethodGet && strings.HasPrefix(p, "/api/v1/users/"):
if !f.users[strings.TrimPrefix(p, "/api/v1/users/")] {
reply(404, map[string]any{"message": "no"})
return
}
reply(200, map[string]any{"active": true})
case r.Method == http.MethodPost && p == "/api/v1/admin/users":
f.users[in["username"].(string)] = true
reply(201, map[string]any{})
case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodGet:
out := []map[string]any{}
for name, id := range f.ids {
out = append(out, map[string]any{"id": id, "name": name})
}
reply(200, out)
case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodPost:
f.next++
name := in["name"].(string)
f.ids[name], f.teams[name] = f.next, in
reply(201, map[string]any{"id": f.next})
default:
http.Error(w, "unexpected "+r.Method+" "+p, 400)
}
}
func (f *teamForge) titles(user string) []string {
f.mu.Lock()
defer f.mu.Unlock()
out := []string{}
for _, k := range f.keys[user] {
out = append(out, k.Title)
}
return out
}
type teamRig struct {
t *testing.T
bus *fakeBus
forge *teamForge
svc *Service
runner *fakeRunner
}
func newTeamRig(t *testing.T) *teamRig {
t.Helper()
forge := newTeamForge()
srv := httptest.NewServer(forge)
t.Cleanup(srv.Close)
dir := t.TempDir()
pw, host, origins := filepath.Join(dir, "pw"), filepath.Join(dir, "host"), filepath.Join(dir, "origins")
for p, c := range map[string]string{pw: "x\n", host: "novox.internal\n", origins: "https://git.novox.be\n"} {
if err := os.WriteFile(p, []byte(c), 0o600); err != nil {
t.Fatal(err)
}
}
runner := &fakeRunner{out: scanned}
svc := NewService(Config{Node: "novox", HostFile: host, OriginsFile: origins, Port: 222,
Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}, Runner: runner, Log: func(string, ...any) {}})
svc.loaded = true
svc.loadedFrom["mesh-build-go"], svc.loadedFrom["mesh-build-node"] = true, true
return &teamRig{t: t, bus: newBus(), forge: forge, svc: svc, runner: runner}
}
func (r *teamRig) pass() []provider.Outcome {
r.t.Helper()
out, err := r.svc.Provider(r.bus).Pass(context.Background())
if err != nil {
r.t.Fatal(err)
}
return out
}
// bind makes module on node bind gitea's forge-account in the assignments mirror.
func (r *teamRig) bind(node, module string) {
r.bus.set(provider.AssignmentsBucket, node+"."+module, []byte(fmt.Sprintf(
`{"node":%q,"module":%q,"provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s"}]}}`,
node, module, strings.ReplaceAll(node+"_"+module, "-", "_"))))
}
func (r *teamRig) unbind(node, module string) {
r.bus.mu.Lock()
delete(r.bus.buckets[provider.AssignmentsBucket], node+"."+module)
r.bus.mu.Unlock()
}
// key reports a key in module's state keys, of the kind and for the account given.
func (r *teamRig) key(module, node, account, kind, public string) {
b, _ := json.Marshal(map[string]string{"node": node, "account": account, "kind": kind, "public": public,
"fingerprint": "SHA256:x", "at": "2026-10-20T10:00:00Z"})
r.svc.SourceKeyChanged(module, node+"."+account, b)
}
const (
pubGo = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG"
pubNode = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN"
pubOps = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO"
)
// 1481/AC1: an entry from mesh-build-go on ace: the forge user nox-builder and the team builders are ensured, the team's
// units are code, read only, on every repository, and the entry's values name nox-builder. The key is registered
// read-only under its title.
func TestABuildModulesEntryIsServedAsNoxBuilderInAReadOnlyTeam(t *testing.T) {
r := newTeamRig(t)
r.bind("ace", "mesh-build-go")
r.key("mesh-build-go", "ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)")
r.pass()
if !r.forge.users["nox-builder"] || r.forge.users["nox-agent"] {
t.Errorf("users = %v, want nox-builder alone", r.forge.users)
}
shape := r.forge.teams["builders"]
if shape == nil {
t.Fatalf("no team builders: %v", r.forge.teams)
}
units, _ := json.Marshal(shape["units_map"])
if shape["permission"] != "read" || shape["includes_all_repositories"] != true || shape["can_create_org_repo"] != false || string(units) != `{"repo.code":"read"}` {
t.Errorf("the team's shape = %v (units %s), want code read on every repository and no other right", shape, units)
}
if got := strings.Join(r.forge.members["builders"], ","); got != "nox-builder" || len(r.forge.members["agents"]) != 0 {
t.Errorf("members = %v", r.forge.members)
}
keys := r.forge.keys["nox-builder"]
if len(keys) != 1 || keys[0].Title != "mesh:ace.mesh-build-go" || !keys[0].ReadOnly {
t.Errorf("nox-builder's keys = %+v, want one read-only key titled mesh:ace.mesh-build-go", keys)
}
e := r.bus.entry(t, "ace.mesh-build-go.forge-account")
if e.Values["user"] != "nox-builder" || e.Values["ssh_alias"] != "mesh-forge" || e.Values["host_key"] != "ssh-ed25519 AAAAC3HOSTKEY" {
t.Errorf("values = %v", e.Values)
}
}
// 1481/AC2: an entry from ssh-client is what it was: nox-agent, the agents team, a key that may write. (The existing
// tests of the provider pass unchanged.)
func TestAnSshClientEntryIsStillTheAgentsWithAKeyThatMayWrite(t *testing.T) {
r := newTeamRig(t)
// the agents team exists, as the forge has it
r.forge.ids["agents"], r.forge.teams["agents"] = 5, map[string]any{}
r.bind("g14", "ssh-client")
r.key("ssh-client", "g14", "nox", "machine", pubA+" nox@g14 (mesh)")
r.pass()
if e := r.bus.entry(t, "g14.ssh-client.forge-account"); e.Values["user"] != "nox-agent" {
t.Errorf("values = %v", e.Values)
}
keys := r.forge.keys["nox-agent"]
if len(keys) != 1 || keys[0].Title != "mesh:g14.nox" || keys[0].ReadOnly {
t.Errorf("nox-agent's keys = %+v, want one key that may write", keys)
}
if r.forge.teams["builders"] != nil || r.forge.users["nox-builder"] {
t.Errorf("a machine's entry made the builders' account or team")
}
if got := strings.Join(r.forge.members["agents"], ","); got != "nox-agent" {
t.Errorf("agents = %v", r.forge.members)
}
}
// 1481/AC3: a builder entry gone: only the keys titled mesh:<node>.mesh-build-go are removed; the user and the team
// stay, and so do the other build module's key on that machine and a key of the operator's.
func TestABuilderEntryGoneRemovesItsOwnKeyAndNothingElse(t *testing.T) {
r := newTeamRig(t)
r.bind("ace", "mesh-build-go")
r.bind("ace", "mesh-build-node")
r.key("mesh-build-go", "ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)")
r.key("mesh-build-node", "ace", "mesh-build-node", "builder", pubNode+" mesh-build-node@ace (mesh)")
r.pass()
r.forge.mu.Lock()
r.forge.keys["nox-builder"] = append(r.forge.keys["nox-builder"], forgeKey{ID: 99, Title: "my own", Key: pubOps})
r.forge.mu.Unlock()
if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-go,mesh:ace.mesh-build-node,my own" {
t.Fatalf("before: %s", got)
}
r.unbind("ace", "mesh-build-go")
r.pass()
if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-node,my own" {
t.Errorf("after: %s, want only the go module's key gone", got)
}
if !r.forge.users["nox-builder"] || r.forge.teams["builders"] == nil || len(r.forge.members["builders"]) != 1 {
t.Errorf("the user or the team went: users %v, members %v", r.forge.users, r.forge.members)
}
if r.bus.has("ace.mesh-build-go.forge-account") {
t.Errorf("the entry stayed")
}
}
// A build module and ssh-client on one machine each have their own entry, keys, user and team.
func TestABuildModuleAndSshClientOnOneMachineDoNotShareAnything(t *testing.T) {
r := newTeamRig(t)
r.forge.ids["agents"], r.forge.teams["agents"] = 5, map[string]any{}
r.bind("ace", "ssh-client")
r.bind("ace", "mesh-build-go")
r.key("ssh-client", "ace", "nox", "machine", pubA+" nox@ace (mesh)")
r.key("mesh-build-go", "ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)")
r.pass()
if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "mesh:ace.nox" {
t.Errorf("nox-agent: %s", got)
}
if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-go" {
t.Errorf("nox-builder: %s", got)
}
// ssh-client's entry gone: nox-builder's key stays.
r.unbind("ace", "ssh-client")
r.pass()
if got := strings.Join(r.forge.titles("nox-builder"), ","); got != "mesh:ace.mesh-build-go" {
t.Errorf("nox-builder after ssh-client left: %s", got)
}
if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "" {
t.Errorf("nox-agent after ssh-client left: %s", got)
}
}
// A build module's entry waits for its own module's builder key: a machine key of the same node, a key of the wrong
// kind in the build module's state, or the other build module's key is none.
func TestABuildModulesEntryWaitsForItsOwnBuilderKey(t *testing.T) {
r := newTeamRig(t)
r.bind("ace", "mesh-build-go")
r.key("ssh-client", "ace", "nox", "machine", pubA) // a machine's key
r.key("mesh-build-go", "ace", "mesh-build-go", "machine", pubGo) // the wrong kind in the build module's state
r.key("mesh-build-node", "ace", "mesh-build-node", "builder", pubNode) // the other build module's
out := r.pass()
if r.bus.has("ace.mesh-build-go.forge-account") || len(r.forge.users) != 0 || len(r.forge.teams) != 0 {
t.Fatalf("something was made without a builder key: users %v teams %v", r.forge.users, r.forge.teams)
}
if len(out) != 1 || !strings.Contains(out[0].Why, "mesh-build-go") || !strings.Contains(out[0].Why, "builder key") {
t.Errorf("the wait should name the module and the key: %+v", out)
}
}
func TestTheKindOfAModulesKeyAndItsForgeUser(t *testing.T) {
for module, want := range map[string]string{"mesh-build-go": "nox-builder", "mesh-build-node": "nox-builder", "ssh-client": "nox-agent", "anything-else": "nox-agent"} {
got, err := (&Service{}).valuesForUser("ace", module)
if err != nil || got != want {
t.Errorf("%s: %q, %v; want %q", module, got, err, want)
}
}
}
modules/gitea/cmd/forge-account-provider/main.go
+22 -13
View File
@@ -4,7 +4,10 @@
// For each machine whose ssh-client binds the provision and has reported its public key (ssh-client's state `keys`),
// it makes the machine's agent the one forge user nox-agent (in the agents team; ADR 0410), registers the key for
// it, and serves where and as whom to reach the forge over ssh: the user, the forge machine's mesh name and port, its
// host key and the two url roots. A public key is no secret, so the provision has no credential.
// host key and the two url roots. A public key is no secret, so the provision has no credential. For each build
// module that binds it and has reported its builder key (its own state `keys`), it does the same as the forge user
// nox-builder, in the builders team that reads the code of every repository and nothing else, with the key read-only
// (ADR 0466 §3).
//
// stdout is the MCP channel; it says what it says on stderr, and never the admin's password.
package main
@@ -22,6 +25,7 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/provider"
"gitea/internal/forgeaccount"
"gitea/internal/forgehttp"
)
@@ -42,11 +46,12 @@ func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte,
return out, nil
}
// stateKeys is ssh-client's state `keys` over the runtime (the manifest's `reads` grants it).
type stateKeys struct{}
// stateKeys is the state `keys` of one module over the runtime (the manifest's `reads` grants it): ssh-client's for
// the machines' keys, a build module's for the builders' (ADR 0466 §3).
type stateKeys struct{ module string }
func (stateKeys) Watch(handle func(key, op string, value []byte, current bool)) error {
return stdio.State("ssh-client.keys").Watch("", func(c stdio.StateChange) error {
func (k stateKeys) Watch(handle func(key, op string, value []byte, current bool)) error {
return stdio.State(k.module+".keys").Watch("", func(c stdio.StateChange) error {
handle(c.Key, c.Op, c.Value, c.Current)
return nil
})
@@ -106,14 +111,18 @@ func main() {
}
}
svc.OnKeys = func(string) { go pass() }
go func() {
if err := svc.WatchKeys(ctx, stateKeys{}); err != nil {
say("the machines' keys are not watched: %v", err)
return
}
say("watching the machines' keys")
pass()
}()
// Each module's keys are watched on their own, and a module whose state never answers (a build module that is not
// installed) holds back its own entries only.
for _, module := range append([]string{SSHClient}, forgeaccount.BuildModules...) {
go func() {
if err := svc.WatchSource(ctx, module, stateKeys{module}); err != nil {
say("the keys of %s are not watched: %v", module, err)
return
}
say("watching the keys of %s", module)
pass()
}()
}
go func() {
svc.KeepRunning(ctx, p.Run)
}()
modules/gitea/cmd/forge-account-provider/service.go
+156 -47
View File
@@ -64,8 +64,11 @@ type Service struct {
mu sync.Mutex
keys map[string]machineKey
loaded bool // the machines' keys have been read: until then a node with none has not reported none
known string // the forge's host key line, as the machines write it into known_hosts
loaded bool // the machines' keys have been read: until then a node with none has not reported none
// loadedFrom says of each build module's state that its keys have been read, as loaded says of ssh-client's: a build
// module's state that never answers (the module is not installed) holds back its own entries and no other.
loadedFrom map[string]bool
known string // the forge's host key line, as the machines write it into known_hosts
bound provider.Bus // the bus the entries are written to (set by Provider)
@@ -76,14 +79,40 @@ type Service struct {
type machineKey struct {
Node, Account, Kind, Public string
// Source is the module whose state the key was read from: ssh-client's for a machine's, a build module's for a
// builder's. Empty is ssh-client's.
Source string
}
// SSHClient is the module whose `keys` state holds the machines' keys (kind machine); the build modules' states hold
// the builders' (kind builder, ADR 0466 §2).
const SSHClient = "ssh-client"
// sourceOf is the module a stored key was read from.
func (k machineKey) sourceOf() string {
if k.Source == "" {
return SSHClient
}
return k.Source
}
// storageKey is the key a source's key is kept under: a machine's as it is, a builder's under its module's name.
func storageKey(source, key string) string {
if source == "" || source == SSHClient {
return key
}
return source + "/" + key
}
// kindOf is the kind of key a consumer module's entry is made for.
func kindOf(module string) string { return forgeaccount.KindOfModule(module) }
// NewService is the service over its environment.
func NewService(cfg Config) *Service {
if cfg.Log == nil {
cfg.Log = func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[gitea forge-account] "+f+"\n", a...) }
}
return &Service{cfg: cfg, keys: map[string]machineKey{}, letGo: map[string]bool{}, Pause: time.Sleep,
return &Service{cfg: cfg, keys: map[string]machineKey{}, letGo: map[string]bool{}, loadedFrom: map[string]bool{}, Pause: time.Sleep,
accounts: &forgeaccount.Accounts{Forge: cfg.Forge, Log: cfg.Log}}
}
@@ -103,38 +132,67 @@ func (s *Service) Provider(bus provider.Bus) *provider.Provider {
}
// errNoKey is a consumer whose machine has reported no key yet: nothing is made, and the client says it waits.
type errNoKey struct{ node string }
type errNoKey struct{ node, module string }
func (e errNoKey) Error() string {
if kind := kindOf(e.module); kind == "builder" {
return "no key from " + e.node + " yet: its " + e.module + " has not reported a builder key, so there is no " +
"entry to make"
}
return "no key from " + e.node + " yet: its ssh-client has not reported a machine key, so there is no account " +
"and no entry to make"
}
// sourceOfModule is the module whose state holds the keys a consumer module's entry is made from: a build module's
// own, ssh-client's for any other consumer.
func sourceOfModule(module string) string {
if kindOf(module) == "builder" {
return module
}
return SSHClient
}
// scopeFor is the title prefix of the keys one entry registered: every key of a machine for a machine's entry, one
// build module's key for a builder's (nox-builder holds the keys of every build module of the mesh).
func scopeFor(kind, node, module string) string {
if kind == "builder" {
return forgeaccount.BuilderScope(node, module)
}
return forgeaccount.MachineScope(node)
}
// Make brings one consumer's account, its team membership and its keys into being, as the machine reported them. It
// leaves what is as asked alone, so it is asked again freely.
func (s *Service) Make(ctx context.Context, c provider.Consumer) error {
s.making.Lock()
defer s.making.Unlock()
node := c.Entry.Consumer
user := forgeaccount.ForgeUser(node, "machine")
module, kind := c.Entry.Module, kindOf(c.Entry.Module)
user := forgeaccount.ForgeUser(node, kind)
if user == "" {
return fmt.Errorf("%q is not a machine's name an account can be named for", node)
}
source := sourceOfModule(module)
s.mu.Lock()
loaded := s.loaded
if source != SSHClient {
loaded = s.loadedFrom[source]
}
s.mu.Unlock()
if !loaded {
// After a restart the first pass can come before the keys arrive: nothing is removed or deleted on that.
return errors.New("the machines' keys are not read yet, so nothing is made, removed or deleted")
return fmt.Errorf("the keys of %s are not read yet, so nothing is made, removed or deleted", source)
}
have := s.keysOf(node)
have := s.keysFrom(source, node, kind)
if len(have) == 0 {
if err := s.releasePerNodeUser(ctx, node); err != nil {
return err
if kind == "machine" {
if err := s.releasePerNodeUser(ctx, node); err != nil {
return err
}
}
// A key the machine no longer reports is no longer wanted (ADR 0400 §2): the mesh's keys go, the operator's
// stay, and the entry that served the machine goes with them.
if _, err := s.accounts.RemoveKeysTitled(ctx, user, forgeaccount.MachineScope(node)); err != nil {
if _, err := s.accounts.RemoveKeysTitled(ctx, user, scopeFor(kind, node, module)); err != nil {
return err
}
if s.bound != nil {
@@ -142,7 +200,7 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error {
return err
}
}
return errNoKey{node}
return errNoKey{node: node, module: module}
}
if _, err := s.hostName(); err != nil {
return err
@@ -154,20 +212,23 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error {
if err := s.accounts.EnsureAccount(ctx, user); err != nil {
return err
}
if err := s.accounts.EnsureInTeam(ctx, user); err != nil {
if err := s.accounts.EnsureInTeamOf(ctx, forgeaccount.TeamOfUser(user), user); err != nil {
return err
}
var wanted []forgeaccount.WantedKey
for _, k := range have {
wanted = append(wanted, forgeaccount.WantedKey{Title: forgeaccount.KeyTitle(k.Node, k.Account), Key: k.Public})
// A builder's key may read and never push (ADR 0466 §3): the key says it beside the team that does.
wanted = append(wanted, forgeaccount.WantedKey{Title: forgeaccount.KeyTitle(k.Node, k.Account), Key: k.Public, ReadOnly: kind == "builder"})
}
// The forge holds one public key on one user only, so the key leaves agent-<node> before it joins nox-agent. That
// is done here, after everything that can fail before it, so the window in which the machine's key is on no user
// is only the span between these two calls; if the add fails, the next pass (at most ten minutes) puts it back.
if err := s.releasePerNodeUser(ctx, node); err != nil {
return err
if kind == "machine" {
if err := s.releasePerNodeUser(ctx, node); err != nil {
return err
}
}
if err := s.accounts.EnsureKeys(ctx, user, forgeaccount.MachineScope(node), wanted); err != nil {
if err := s.accounts.EnsureKeys(ctx, user, scopeFor(kind, node, module), wanted); err != nil {
return err
}
s.mu.Lock()
@@ -176,20 +237,25 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error {
return nil
}
// Gone removes the keys the mesh registered for a machine that no longer binds the forge. The account stays: it may
// Gone removes the keys the mesh registered for a machine (or, for a build module, the module's key on that machine)
// that no longer binds the forge. The account stays: it may
// hold what the machine's agent wrote, and the keys of the operator's own that were added by hand are not touched
// (ADR 0399 §2).
func (s *Service) Gone(ctx context.Context, e provider.Entry) error {
s.making.Lock()
defer s.making.Unlock()
user := forgeaccount.ForgeUser(e.Consumer, "machine")
kind := kindOf(e.Module)
user := forgeaccount.ForgeUser(e.Consumer, kind)
if user == "" {
return nil
}
if err := s.releasePerNodeUser(ctx, e.Consumer); err != nil {
return err
if kind == "machine" {
if err := s.releasePerNodeUser(ctx, e.Consumer); err != nil {
return err
}
}
_, err := s.accounts.RemoveKeysTitled(ctx, user, forgeaccount.MachineScope(e.Consumer))
// A builder's entry gone removes that entry's key only: the user and the team serve the other machines.
_, err := s.accounts.RemoveKeysTitled(ctx, user, scopeFor(kind, e.Consumer, e.Module))
return err
}
@@ -220,17 +286,24 @@ func (s *Service) releasePerNodeUser(ctx context.Context, node string) error {
}
// KeyChanged takes in one machine key as the bus keeps it, and tells OnKeys its node.
func (s *Service) KeyChanged(key string, value []byte) {
if node := s.store(key, value); node != "" && s.OnKeys != nil {
func (s *Service) KeyChanged(key string, value []byte) { s.SourceKeyChanged(SSHClient, key, value) }
// KeyGone forgets one machine key, and tells OnKeys its node.
func (s *Service) KeyGone(key string) { s.SourceKeyGone(SSHClient, key) }
// SourceKeyChanged takes in one key of the state `keys` of the module source (ssh-client's, or a build module's), and
// tells OnKeys its node.
func (s *Service) SourceKeyChanged(source, key string, value []byte) {
if node := s.storeFrom(source, key, value); node != "" && s.OnKeys != nil {
s.OnKeys(node)
}
}
// KeyGone forgets one machine key, and tells OnKeys its node.
func (s *Service) KeyGone(key string) {
// SourceKeyGone forgets one key of the module source's state, and tells OnKeys its node.
func (s *Service) SourceKeyGone(source, key string) {
node, _, _ := strings.Cut(key, ".")
s.mu.Lock()
delete(s.keys, key)
delete(s.keys, storageKey(source, key))
s.mu.Unlock()
if s.OnKeys != nil {
s.OnKeys(node)
@@ -240,24 +313,38 @@ func (s *Service) KeyGone(key string) {
// store keeps a machine key that reads as one: its node and account the key's own, and a public half that is a
// public key line and nothing private. It answers the node, or "" for a value it did not keep (and drops what was
// kept under that key, since the key now says something else).
func (s *Service) store(key string, value []byte) string {
func (s *Service) store(key string, value []byte) string { return s.storeFrom(SSHClient, key, value) }
// storeFrom is store for a key of the module source's state.
func (s *Service) storeFrom(source, key string, value []byte) string {
node, account, _ := strings.Cut(key, ".")
stored := storageKey(source, key)
var k machineKey
var raw struct {
Node, Account, Kind, Public string
}
err := json.Unmarshal(value, &raw)
k = machineKey(raw)
k = machineKey{Node: raw.Node, Account: raw.Account, Kind: raw.Kind, Public: raw.Public}
if source != SSHClient {
k.Source = source
}
s.mu.Lock()
defer s.mu.Unlock()
if err != nil || k.Node != node || k.Account != account || !publicKeyLine(k.Public) {
if _, had := s.keys[key]; had {
delete(s.keys, key)
}
s.cfg.Log("the key %s is not a machine's public key; it is not registered", key)
if want := kindOf(source); err == nil && k.Kind != want && k.Node == node && k.Account == account {
// Each state holds one kind: ssh-client's the machines', a build module's the builders' (ADR 0466 §2). The other
// kind in either is not this entry's to serve, and is said.
delete(s.keys, stored)
s.cfg.Log("the key %s in the state keys of %s is of kind %q, not %q; it is ignored", key, source, k.Kind, want)
return ""
}
s.keys[key] = k
if err != nil || k.Node != node || k.Account != account || !publicKeyLine(k.Public) {
if _, had := s.keys[stored]; had {
delete(s.keys, stored)
}
s.cfg.Log("the key %s of %s is not a public key that can be registered; it is not registered", key, source)
return ""
}
s.keys[stored] = k
return node
}
@@ -267,12 +354,15 @@ func publicKeyLine(line string) bool {
}
// keysOf are the machine keys one node reported, by key.
func (s *Service) keysOf(node string) []machineKey {
func (s *Service) keysOf(node string) []machineKey { return s.keysFrom(SSHClient, node, "machine") }
// keysFrom are the keys of the kind that one node reported in the module source's state, by key.
func (s *Service) keysFrom(source, node, kind string) []machineKey {
s.mu.Lock()
defer s.mu.Unlock()
var names []string
for key, k := range s.keys {
if k.Node == node && k.Kind == "machine" && strings.HasPrefix(key, node+".") {
if k.Node == node && k.Kind == kind && k.sourceOf() == source && strings.HasPrefix(key, storageKey(source, node+".")) {
names = append(names, key)
}
}
@@ -284,27 +374,37 @@ func (s *Service) keysOf(node string) []machineKey {
return out
}
// WatchKeys keeps the machines' keys as the bus holds them, asking again until the runtime answers (the channel to it
// opens as the bundle starts, and the state of ssh-client may arrive after this one). A change after the first
// values is told to OnKeys; the first values are not, since one pass answers them all.
// WatchKeys keeps the machines' keys as the bus holds them (ssh-client's state `keys`), asking again until the runtime
// answers (the channel to it opens as the bundle starts, and the state of ssh-client may arrive after this one). A
// change after the first values is told to OnKeys; the first values are not, since one pass answers them all.
func (s *Service) WatchKeys(ctx context.Context, src KeySource) error {
return s.WatchSource(ctx, SSHClient, src)
}
// WatchSource is WatchKeys for the state `keys` of any module whose keys the provider registers: ssh-client's, or a
// build module's (ADR 0466 §3). Once its first values are in, the entries made from them are no longer held back.
func (s *Service) WatchSource(ctx context.Context, source string, src KeySource) error {
waits := []time.Duration{2 * time.Second, 5 * time.Second, 10 * time.Second, 30 * time.Second}
for attempt := 0; ; attempt++ {
err := src.Watch(func(key, op string, value []byte, current bool) {
switch {
case op != "put" && current:
case op != "put":
s.KeyGone(key)
s.SourceKeyGone(source, key)
case current:
s.store(key, value)
s.storeFrom(source, key, value)
default:
s.KeyChanged(key, value)
s.SourceKeyChanged(source, key, value)
}
})
if err == nil {
// The watch answers once the current values are all handled (stdio KeptState.Watch).
s.mu.Lock()
s.loaded = true
if source == SSHClient {
s.loaded = true
} else {
s.loadedFrom[source] = true
}
s.mu.Unlock()
return nil
}
@@ -312,7 +412,7 @@ func (s *Service) WatchKeys(ctx context.Context, src KeySource) error {
if attempt < len(waits) {
pause = waits[attempt]
}
s.cfg.Log("the machines' keys cannot be watched yet (%v); asking again in %s", err, pause)
s.cfg.Log("the keys of %s cannot be watched yet (%v); asking again in %s", source, err, pause)
if ctx.Err() != nil {
return ctx.Err()
}
@@ -377,9 +477,18 @@ func knownHostLine(scanned, host string, port int) (string, error) {
return "", errors.New("the forge's sshd gave no ed25519 host key")
}
// valuesForUser is the forge user a consumer module's entry is served as.
func (s *Service) valuesForUser(node, module string) (string, error) {
user := forgeaccount.ForgeUser(node, kindOf(module))
if user == "" {
return "", fmt.Errorf("%q is not a machine's name an account can be named for", node)
}
return user, nil
}
// valuesFor are the values served to one machine's consumer (ADR 0400 §3, §4).
func (s *Service) valuesFor(node string) (map[string]any, error) {
user := forgeaccount.ForgeUser(node, "machine")
func (s *Service) valuesFor(node, module string) (map[string]any, error) {
user := forgeaccount.ForgeUser(node, kindOf(module))
host, err := s.hostName()
if err != nil {
return nil, err
@@ -421,7 +530,7 @@ func (b *valuesBus) Put(ctx context.Context, bucket, key string, value []byte) e
return err
}
if e.Refused == "" {
extra, err := b.s.valuesFor(e.Consumer)
extra, err := b.s.valuesFor(e.Consumer, e.Module)
if err != nil {
return err
}
modules/gitea/cmd/forge-account-provider/watch_test.go
+162
View File
@@ -0,0 +1,162 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"slices"
"strings"
"sync"
"testing"
"time"
)
// Issue 1482 (novox/hq ADR 0466 §3): the provider watches the build modules' state `keys` beside ssh-client's, and
// registers each builder entry read-only on nox-builder.
func keyJSON(node, account, kind, public string) []byte {
b, _ := json.Marshal(map[string]string{"node": node, "account": account, "kind": kind, "public": public,
"fingerprint": "SHA256:x", "at": "2026-10-20T10:00:00Z"})
return b
}
// watched is a team rig whose keys come through watches of the modules' states, as in the running provider.
type watched struct {
*teamRig
sources map[string]*fakeKeys
passes *int
logs *[]string
mu *sync.Mutex
}
func newWatched(t *testing.T) *watched {
r := newTeamRig(t)
r.svc.loaded, r.svc.loadedFrom = false, map[string]bool{}
r.svc.Pause = func(time.Duration) {}
n, logs, mu := 0, []string{}, &sync.Mutex{}
r.svc.OnKeys = func(string) { n++ }
r.svc.cfg.Log = func(f string, a ...any) { mu.Lock(); logs = append(logs, fmt.Sprintf(f, a...)); mu.Unlock() }
w := &watched{teamRig: r, sources: map[string]*fakeKeys{}, passes: &n, logs: &logs, mu: mu}
for _, module := range []string{"ssh-client", "mesh-build-go", "mesh-build-node"} {
w.sources[module] = &fakeKeys{}
if err := r.svc.WatchSource(context.Background(), module, w.sources[module]); err != nil {
t.Fatal(err)
}
}
return w
}
func (w *watched) said() string {
w.mu.Lock()
defer w.mu.Unlock()
return strings.Join(*w.logs, "\n")
}
// 1482/AC1: a mesh-build-go.keys entry ace.mesh-build-go of kind builder: its public half is registered read-only on
// nox-builder titled mesh:ace.mesh-build-go.
func TestABuilderEntryOfABuildModulesStateIsRegisteredReadOnlyOnNoxBuilder(t *testing.T) {
w := newWatched(t)
w.bind("ace", "mesh-build-go")
w.sources["mesh-build-go"].send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "builder", pubGo+" mesh-build-go@ace (mesh)"), true)
w.pass()
keys := w.forge.keys["nox-builder"]
if len(keys) != 1 || keys[0].Title != "mesh:ace.mesh-build-go" || !keys[0].ReadOnly || keys[0].Key != pubGo+" mesh-build-go@ace (mesh)" {
t.Errorf("nox-builder's keys = %+v", keys)
}
if !w.svc.loadedFrom["mesh-build-go"] {
t.Errorf("the state was watched and its keys are not counted as read")
}
}
// 1482/AC2: that entry changed: the old key is removed and the new one registered; deleted: the key is removed and no key
// titled otherwise is touched.
func TestAChangedBuilderEntryReplacesItsKeyAndADeletedOneRemovesIt(t *testing.T) {
w := newWatched(t)
w.bind("ace", "mesh-build-go")
src := w.sources["mesh-build-go"]
src.send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "builder", pubGo), true)
w.pass()
w.forge.mu.Lock()
w.forge.keys["nox-builder"] = append(w.forge.keys["nox-builder"], forgeKey{ID: 99, Title: "my own", Key: pubOps})
w.forge.mu.Unlock()
before := *w.passes
src.send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "builder", pubNode), false)
if *w.passes != before+1 {
t.Errorf("a change asked for %d passes, want 1", *w.passes-before)
}
w.pass()
got := w.forge.keys["nox-builder"]
if len(got) != 2 || got[0].Title != "my own" || got[1].Title != "mesh:ace.mesh-build-go" || got[1].Key != pubNode || !got[1].ReadOnly {
t.Errorf("after the change: %+v", got)
}
src.send("ace.mesh-build-go", "delete", nil, false)
w.pass()
if got := strings.Join(w.forge.titles("nox-builder"), ","); got != "my own" {
t.Errorf("after the delete: %s, want only the operator's key", got)
}
}
// 1482/AC3: an entry of kind machine in a build module's state, or of kind builder in ssh-client's, is ignored and
// said in the provider's log.
func TestAKeyOfTheWrongKindForItsStateIsIgnoredAndSaid(t *testing.T) {
w := newWatched(t)
w.bind("ace", "mesh-build-go")
w.bind("g14", "ssh-client")
w.sources["mesh-build-go"].send("ace.mesh-build-go", "put", keyJSON("ace", "mesh-build-go", "machine", pubGo), true)
w.sources["ssh-client"].send("g14.nox", "put", keyJSON("g14", "nox", "builder", pubA), true)
w.pass()
if len(w.forge.users) != 0 || w.bus.has("ace.mesh-build-go.forge-account") || w.bus.has("g14.ssh-client.forge-account") {
t.Errorf("a key of the wrong kind was served: users %v", w.forge.users)
}
said := w.said()
for _, want := range []string{`ace.mesh-build-go in the state keys of mesh-build-go is of kind "machine", not "builder"`, `g14.nox in the state keys of ssh-client is of kind "builder", not "machine"`} {
if !strings.Contains(said, want) {
t.Errorf("the log lacks %q:\n%s", want, said)
}
}
}
// A build module's state that never answers holds back its own entries, never a machine's.
func TestABuildModulesStateThatNeverAnswersHoldsBackOnlyItsOwnEntries(t *testing.T) {
w := newWatched(t)
w.svc.loadedFrom = map[string]bool{} // mesh-build-go's watch has not answered
w.bind("ace", "mesh-build-go")
w.bind("g14", "ssh-client")
w.sources["ssh-client"].send("g14.nox", "put", keyJSON("g14", "nox", "machine", pubA+" nox@g14 (mesh)"), true)
out := w.pass()
if !w.bus.has("g14.ssh-client.forge-account") {
t.Errorf("a machine's entry waited for a build module's state")
}
waits := 0
for _, o := range out {
if strings.Contains(o.Key, "mesh-build-go") && strings.Contains(o.Why, "keys of mesh-build-go are not read yet") {
waits++
}
}
if waits != 1 {
t.Errorf("outcomes = %+v", out)
}
}
// 1482/AC4 (the provider's half): the manifest reads the states of both build modules beside ssh-client's and the
// controller's assignments; the provider watches exactly the modules the code names as builders.
func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Reads []string `json:"reads"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys"} {
if !slices.Contains(m.Reads, state) {
t.Errorf("gitea's reads %v lack %q", m.Reads, state)
}
}
}
modules/gitea/internal/forgeaccount/account.go
+79 -17
View File
@@ -1,7 +1,8 @@
// Package forgeaccount is the forge accounts the mesh makes for its agents (novox/hq ADR 0365 clauses 1 and 3, ADR
// 0398 §1, ADR 0399, ADR 0400; issue 955): an account of the agent's own, no administrator, in the agents write
// team. Both the token tool (cmd/agent-token) and the forge-account provider use it, so there is one place that says
// what an agent's account is.
// what an agent's account is. It also makes the one account of the build modules' keys, nox-builder, in the builders
// team that reads the code of every repository and nothing else (ADR 0466 §3; issue 1481).
package forgeaccount
import (
@@ -26,8 +27,23 @@ const (
TeamName = "agents"
)
// teamShape is what the agents team grants: write on code, issues and pull requests of every repository of the
// BuildersTeamName is the team of the build modules' account (ADR 0466 §3).
const BuildersTeamName = "builders"
// Team is a team of the organisation as the mesh keeps it: its name and the shape it is put back to.
type Team struct {
Name string
Shape func() map[string]any
}
// AgentsTeam is the team of the machines' agents: write on code, issues and pull requests of every repository of the
// organisation and no more, the shape of the forge tools' team (tools-account.ts).
var AgentsTeam = Team{Name: TeamName, Shape: teamShape}
// BuildersTeam is the team of the build modules' account: it may read the code of every repository of the organisation
// and has no other right (ADR 0466 §3).
var BuildersTeam = Team{Name: BuildersTeamName, Shape: buildersTeamShape}
func teamShape() map[string]any {
return map[string]any{
"name": TeamName, "permission": "write", "includes_all_repositories": true, "can_create_org_repo": false,
@@ -35,6 +51,13 @@ func teamShape() map[string]any {
}
}
func buildersTeamShape() map[string]any {
return map[string]any{
"name": BuildersTeamName, "permission": "read", "includes_all_repositories": true, "can_create_org_repo": false,
"units_map": map[string]string{"repo.code": "read"},
}
}
// User is gitea's account as the admin reads it.
type User struct {
IsAdmin bool `json:"is_admin"`
@@ -87,35 +110,41 @@ func (m *Accounts) EnsureAccount(ctx context.Context, account string) error {
// EnsureInTeam makes the agents team when there is none, puts a team that is there back to its shape, and adds the
// account to it; an account already in it stays (ADR 0398 §1, issue 948).
func (m *Accounts) EnsureInTeam(ctx context.Context, account string) error {
id, err := m.teamID(ctx)
return m.EnsureInTeamOf(ctx, AgentsTeam, account)
}
// EnsureInTeamOf is EnsureInTeam for any team of the mesh's: it makes the team when there is none, puts a team that is
// there back to its shape, and adds the account to it.
func (m *Accounts) EnsureInTeamOf(ctx context.Context, team Team, account string) error {
id, err := m.teamID(ctx, team.Name)
if err != nil {
return err
}
if id != 0 {
if err := m.Forge.SendJSON(ctx, http.MethodPatch, fmt.Sprintf("/api/v1/teams/%d", id), teamShape(), nil); err != nil {
return fmt.Errorf("the team %s could not be put back to its shape: %w", TeamName, err)
if err := m.Forge.SendJSON(ctx, http.MethodPatch, fmt.Sprintf("/api/v1/teams/%d", id), team.Shape(), nil); err != nil {
return fmt.Errorf("the team %s could not be put back to its shape: %w", team.Name, err)
}
} else {
var made struct {
ID int64 `json:"id"`
}
if err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/orgs/"+TeamOrg+"/teams", teamShape(), &made); err != nil {
return fmt.Errorf("the team %s could not be made: %w", TeamName, err)
if err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/orgs/"+TeamOrg+"/teams", team.Shape(), &made); err != nil {
return fmt.Errorf("the team %s could not be made: %w", team.Name, err)
}
id = made.ID
m.say("made the forge team %s in %s: write on every repository", TeamName, TeamOrg)
m.say("made the forge team %s in %s", team.Name, TeamOrg)
}
if id == 0 {
return fmt.Errorf("the forge answered no id for the team %s", TeamName)
return fmt.Errorf("the forge answered no id for the team %s", team.Name)
}
if err := m.Forge.SendJSON(ctx, http.MethodPut, fmt.Sprintf("/api/v1/teams/%d/members/%s", id, account), nil, nil); err != nil {
return fmt.Errorf("the account %s could not be added to the team %s: %w", account, TeamName, err)
return fmt.Errorf("the account %s could not be added to the team %s: %w", account, team.Name, err)
}
return nil
}
// teamID is the agents team's id, or 0 when there is none.
func (m *Accounts) teamID(ctx context.Context) (int64, error) {
// teamID is the id of the team of that name, or 0 when there is none.
func (m *Accounts) teamID(ctx context.Context, name string) (int64, error) {
var id int64
for page := 1; page <= 20 && id == 0; page++ {
var teams []struct {
@@ -126,7 +155,7 @@ func (m *Accounts) teamID(ctx context.Context) (int64, error) {
return 0, fmt.Errorf("the teams of %s could not be read: %w", TeamOrg, err)
}
for _, t := range teams {
if t.Name == TeamName {
if t.Name == name {
id = t.ID
}
}
@@ -160,16 +189,49 @@ var NodeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`)
// AgentUser is the one forge user of every machine's agents (ADR 0410 §1), in the agents write team.
const AgentUser = "nox-agent"
// ForgeUser is the forge user of a machine's key of the given kind (ADR 0410 §1): `nox-agent` for every machine
// and account (kind machine). A kind not built yet has none, and the answer is empty. The machine's name must still
// be one an account could be named for.
// BuilderUser is the one forge user of every build module's key (ADR 0466 §3), in the builders team: it reads and
// pushes nothing.
const BuilderUser = "nox-builder"
// BuildModules are the modules whose keys are the builders' (kind builder), by their name, which is also the name of
// the account each runs as (ADR 0466 §1).
var BuildModules = []string{"mesh-build-go", "mesh-build-node"}
// KindOfModule is the kind of key a consumer module's entry is made for: builder for a build module, machine for any
// other (ssh-client's).
func KindOfModule(module string) string {
for _, m := range BuildModules {
if m == module {
return "builder"
}
}
return "machine"
}
// ForgeUser is the forge user of a key of the given kind (ADR 0410 §1, ADR 0466 §3): `nox-agent` for every machine
// and account (kind machine), `nox-builder` for every build module (kind builder). A kind not built yet has none, and
// the answer is empty. The machine's name must still be one an account could be named for.
func ForgeUser(node, kind string) string {
if kind == "machine" && NodeName.MatchString(node) {
if !NodeName.MatchString(node) {
return ""
}
switch kind {
case "machine":
return AgentUser
case "builder":
return BuilderUser
}
return ""
}
// TeamOfUser is the team a forge user of the mesh is in.
func TeamOfUser(user string) Team {
if user == BuilderUser {
return BuildersTeam
}
return AgentsTeam
}
// NodeAccount is the forge account the token tool (cmd/agent-token) keeps for a node's agent: `agent-<node>` (ADR
// 0365 §1). It is not the forge user of the machine's ssh key (ForgeUser), and its behaviour is unchanged.
func NodeAccount(node string) string { return AccountPrefix + node }
modules/gitea/internal/forgeaccount/builder_test.go
+46
View File
@@ -0,0 +1,46 @@
package forgeaccount
import "testing"
// Issue 1481 (novox/hq ADR 0466 §3): the build modules' forge user, team and key scope.
func TestTheBuildModulesHaveOneForgeUserAndTheMachinesKeepTheirs(t *testing.T) {
for kind, want := range map[string]string{"machine": "nox-agent", "builder": "nox-builder", "session": ""} {
if got := ForgeUser("ace", kind); got != want {
t.Errorf("ForgeUser(ace, %s) = %q, want %q", kind, got, want)
}
}
if ForgeUser("Not A Node", "builder") != "" {
t.Errorf("a name no account could be named for got a user")
}
for module, want := range map[string]string{"mesh-build-go": "builder", "mesh-build-node": "builder", "ssh-client": "machine", "": "machine"} {
if got := KindOfModule(module); got != want {
t.Errorf("KindOfModule(%q) = %s, want %s", module, got, want)
}
}
if TeamOfUser(BuilderUser).Name != "builders" || TeamOfUser(AgentUser).Name != "agents" {
t.Errorf("teams: %s and %s", TeamOfUser(BuilderUser).Name, TeamOfUser(AgentUser).Name)
}
}
// A build module's scope names the module, so one module's pass does not remove the other's key on the same machine;
// the machine's scope is every key of the machine.
func TestABuilderScopeNamesTheModule(t *testing.T) {
if got := BuilderScope("ace", "mesh-build-go"); got != "mesh:ace.mesh-build-go" {
t.Errorf("scope = %q", got)
}
if BuilderScope("ace", "mesh-build-go") == BuilderScope("ace", "mesh-build-node") {
t.Errorf("two build modules share a scope")
}
if got := KeyTitle("ace", "mesh-build-go"); got != BuilderScope("ace", "mesh-build-go") {
t.Errorf("the key's title %q is not inside the scope", got)
}
}
// The builders team reads the code of every repository of the organisation and has no other right.
func TestTheBuildersTeamReadsCodeOnly(t *testing.T) {
s := BuildersTeam.Shape()
units, _ := s["units_map"].(map[string]string)
if s["permission"] != "read" || s["includes_all_repositories"] != true || s["can_create_org_repo"] != false || len(units) != 1 || units["repo.code"] != "read" {
t.Errorf("shape = %v", s)
}
}
modules/gitea/internal/forgeaccount/keys.go
+9 -2
View File
@@ -22,6 +22,8 @@ type WantedKey struct {
Title string
// Key is the public key line: type, base64 and a comment.
Key string
// ReadOnly registers the key as one that may not push (a builder's, ADR 0466 §3).
ReadOnly bool
}
// Key is a key the forge holds for an account.
@@ -56,10 +58,10 @@ func (m *Accounts) Keys(ctx context.Context, user string) ([]Key, error) {
return all, nil
}
// AddKey registers a key for the account, one it may write with.
// AddKey registers a key for the account, one it may write with unless the key is wanted read-only.
func (m *Accounts) AddKey(ctx context.Context, user string, k WantedKey) error {
err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/admin/users/"+user+"/keys",
map[string]any{"title": k.Title, "key": k.Key, "read_only": false}, nil)
map[string]any{"title": k.Title, "key": k.Key, "read_only": k.ReadOnly}, nil)
if err != nil {
return fmt.Errorf("the key %s could not be added for %s: %w", k.Title, user, err)
}
@@ -78,6 +80,11 @@ func (m *Accounts) DeleteKey(ctx context.Context, user string, id int64) error {
// MachineScope is the title prefix of one machine's keys, `mesh:<node>.`: what a pass for that machine may remove.
func MachineScope(node string) string { return TitlePrefix + node + "." }
// BuilderScope is the title prefix of one build module's key on one machine, `mesh:<node>.<module>`: what a pass for
// that module's entry may remove. It names the module, since nox-builder holds the keys of both build modules of a
// machine and one's pass must not remove the other's.
func BuilderScope(node, module string) string { return TitlePrefix + node + "." + module }
// EnsureKeys registers each wanted key the account does not hold and removes the keys the mesh registered
// for one machine (title starting scope, `mesh:<node>.`) that are no longer wanted. One forge user holds every
// machine's keys (ADR 0410 §2), so a key under another machine's title is never removed here. A key whose title does not start `mesh:` is the operator's and
modules/gitea/module.json
+3 -1
View File
@@ -540,6 +540,8 @@
],
"reads": [
"mesh-controller.assignments",
"ssh-client.keys"
"ssh-client.keys",
"mesh-build-go.keys",
"mesh-build-node.keys"
]
}
modules/mesh-build-go/README.md
+88
View File
@@ -0,0 +1,88 @@
# mesh-build-go — the Go build module
Design 47, release R4c-3 (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470). The module that builds and checks Go
code on a build machine, as an account of its own (`runs-as`, no login shell, home at mode 0700), with the
toolchain installed as the packages it declares (`go`, `git`, `openssh`, `postgresql`) and the capability
`user-namespaces`. It claims no seat: it is assigned to the build machines (ADR 0469 §6) and runs dark until the
builder uses it.
## The recipe
`cmd/mesh-build-go/recipe.go` names the one Go version the module builds with (`RecipeGo`), the postgres major
version the integration level tests with, and the recipe's own version. The recipe version is a hash of those and of
the nats-server library's version, read from the program's build information, so a change of any of them moves it.
## The guard and the health
At start, every ten minutes, before every step and at every health look, the module reads the installed Go
(`go version`) and writes it with the recipe version to `<node>.go` of its `toolchains` state. A machine whose Go
is not the recipe's refuses every step, the step ending `refused` on its key in `steps` with both versions named,
and `build_go_health` (the module's `runtime-health`) answers unhealthy with the same text, so the controller raises
the machine's condition. Moving to a new Go is a change of `RecipeGo`.
## Tools
`build_go_status`, `build_go_health` and `build_go_fetch` are served now; the other step tools (`build_go_test`,
`build_go_digest`, `build_go_build`) are added by their own issues to `StepTools` and run behind the guard: a long
step answers at once with its id and ends on `<node>.<id>` of `steps` (ADR 0469 §2); the digest answers in its call.
## Fetch
`build_go_fetch {repository, commit, ref?}` fetches the commit over SSH as the module's account (`mesh-forge` is the
forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the
work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor,
with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an
environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day
are removed.
## Test
`build_go_test {workspace, level, packages?}` runs the tests of a fetched workspace by Go's convention, one level at a
time. The unit level is `go test -json` over every package without the `integration` build tag, in each directory
with a `go.mod`, judged by its one bound: its running time, that of the slowest package (compile not counted), within
five seconds. The step ends passed when the level ran, and its result says whether the level did: `passed`, the
packages with their times, the tests that failed with their first lines of output, the five slowest tests, and `why`
when it failed (a failed test, a build that failed) or ran over a bound. `passed` is the tests' own verdict: a level over its bound still passes and says `over_bound` (and `over_which`, setup or run) with the slowest tests, and the checker judges the bounds (ADR 0453 §9). A step ends failed only when go test could not
run. A repository with no tests at the level has none, and the result says so: that is a pass. `testdata/captured` holds
the output of the real go for the fixture repositories in `testdata/repos`; the unit tests read it, and the
integration-tagged test runs the real go on the fixtures.
## The integration level
`build_go_test` with `level: integration` runs `go test -tags integration` against a store and a bus started for the
run alone (novox/hq ADR 0470). The store is a postgres of the installed package: `initdb` and `pg_ctl` into
`<work>/<workspace>/run-<id>/pg`, as the module's account, listening on a unix socket there and on no TCP address, with
durability off; the tests get `MESH_TEST_POSTGRES` (the socket directory percent-encoded in the URL, because the
controller's tests cut the URL at its last slash). The bus is a nats-server from the Go library at the version this
module's `go.mod` names, run by this same binary started as `mesh-build-go bus <dir>` (`bus.go`: JetStream on, a
loopback port of its own, its address on the first line, stopped when its input closes); the tests get `MESH_TEST_NATS`.
Both run inside the step, are stopped, and the run's directory is removed whatever the tests did.
Setup (compiling the tests, the store, the bus) is judged within 30 s and the tests' running time (the wall time of
`go test`) within 60 s, apart; a level over either says `over_bound` and names the bound, with the slowest tests for the running
time. A machine whose installed postgres major is not the recipe's refuses this level, naming both; the unit level
still runs. The result names the postgres major and the nats-server version it ran against. The mesh's own store major
is not named: no state or grant gives it to this module (ADR 0470 §4 asks for it when it differs), so only the recipe's
is said until a source exists.
## Digest
`build_go_digest {workspace, level}` answers in its call, for each package of the workspace that has tests at the level,
one digest of everything those tests read (ADR 0445 §5): the files, by content and under their path in the tree, of the
package and every package its test binary reaches (`go list -test -deps`) that lies in the module's own tree; the version
and checksum of every module outside it; the package's own `testdata/` files; and the Go version, the level, the build tags
and, at the integration level, the postgres major and the nats-server version. Packages of the standard library are the Go
version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors
is listed and tested from its vendor directory (`-mod=vendor`).
## Build
`build_go_build {workspace, module, artifact}` builds one Go bundle artifact of a module of the workspace with the build
agent's recipe (mesh-controller's `toolchain.go`, the Go entry, copied in `testdata/captured/toolchain-go-entry.txt`, and
`builder.go`): `CGO_ENABLED=0`, `GOFLAGS=-trimpath -buildvcs=false` (with `-mod=mod`, or `-mod=vendor` for a directory
that vendors), `go build -ldflags "-s -w -X main.builtFor=<system>" -o .mesh-build/<artifact>/<binary> ./<from>` in the
module's directory, by the installed Go (`GOTOOLCHAIN=local`) from the module cache the fetch filled and no network
(`GOPROXY=off`). The binary is named as the build agent names it. The output directory is packed with mesh-sdk's `Pack`,
so the digest is the build agent's for the same tree. The step ends with the artifact, the binary, the digest, the size and
the path of the archive, which stays in the workspace's `artifacts` directory: publishing it is the builder's, in a later
release. An artifact that is not a Go bundle fails the step naming it.
modules/mesh-build-go/cmd/mesh-build-go/build.go
+164
View File
@@ -0,0 +1,164 @@
package main
// The build step (novox/hq ADR 0405 §3 §7, ADR 0453 §7; issue 1487): a Go bundle artifact of a module of a fetched
// workspace, built with today's recipe — the build agent's, mesh-controller's internal/builder/toolchain.go (the Go
// entry) and builder.go (compile, binaryName, pack) — by the installed Go and no network, packed with mesh-sdk's Pack
// so that its digest is the build agent's for the same tree. The archive is left in the workspace; publishing it is
// the builder's, in a later release.
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The Go recipe of the build agent (toolchain.go, the entry of language go): static, trimmed of its own paths and of
// the commit stamp, stripped of symbols and debug info, and told the system it was built for.
const (
recipeFlags = "-trimpath -buildvcs=false"
recipeLinkerFlags = "-s -w"
recipeSystemStamp = "main.builtFor"
// outDir is where a build leaves an artifact in the module's directory (builder.Out).
outDir = ".mesh-build"
)
// BuildResult is what a passed build step says.
type BuildResult struct {
Artifact string `json:"artifact"`
Binary string `json:"binary"`
Digest string `json:"digest"`
Size int `json:"size"`
// Path is the archive, in the workspace's directory of the work directory.
Path string `json:"path"`
}
func (m *Module) buildTool() StepTool {
return StepTool{
Name: "build_go_build",
Description: "Build one Go bundle artifact of a module of a fetched workspace with the build agent's recipe, by the " +
"installed Go and no network, and pack it. Answers a step id at once; the step ends on its key with the " +
"artifact's digest, size and the archive's path.",
Input: map[string]any{
"type": "object",
"properties": map[string]any{
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"},
"artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"},
},
"required": []string{"workspace", "module", "artifact"},
},
Long: true,
Run: m.build,
}
}
// binaryName is what a compiled bundle's executable is called (builder.go binaryName): what the artifact says, or the
// name of the package it is built from, or the artifact's.
func binaryName(a catalogue.Artifact) string {
if name := catalogue.BinaryOf(a); name != "" {
return name
}
if name := strings.TrimSpace(a.Binary); name != "" {
return name
}
if from := strings.Trim(a.From, "./"); from != "" {
return filepath.Base(from)
}
return a.Name
}
func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
workspace, _ := args["workspace"].(string)
module, _ := args["module"].(string)
name, _ := args["artifact"].(string)
if !workspaceOK.MatchString(workspace) {
return nil, fmt.Errorf("%q is not a workspace id", workspace)
}
tree := filepath.Join(m.Work, workspace, "tree")
dir := filepath.Join(tree, filepath.FromSlash(module))
if rel, err := filepath.Rel(tree, dir); err != nil || strings.HasPrefix(rel, "..") {
return nil, fmt.Errorf("the module directory %q lies outside the workspace", module)
}
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
if err != nil {
return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err)
}
manifest, err := catalogue.ParseManifest(raw)
if err != nil {
return nil, fmt.Errorf("the manifest in %q: %w", module, err)
}
var art *catalogue.Artifact
names := []string{}
if manifest.Build != nil {
for i := range manifest.Build.Artifacts {
names = append(names, manifest.Build.Artifacts[i].Name)
if manifest.Build.Artifacts[i].Name == name {
art = &manifest.Build.Artifacts[i]
}
}
}
if art == nil {
return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", "))
}
if art.Kind != "bundle" || art.Language != "go" {
return nil, fmt.Errorf("the artifact %s of %s is a %s of language %q: this module builds Go bundles", name, manifest.Module, art.Kind, art.Language)
}
if strings.TrimSpace(art.From) == "" {
return nil, fmt.Errorf("the artifact %s of %s says nothing it is built from", name, manifest.Module)
}
// Its own output directory, made empty: a build of the same tree twice is one build.
out := filepath.Join(dir, outDir, art.Name)
if err := os.RemoveAll(out); err != nil {
return nil, err
}
if err := os.MkdirAll(out, 0o755); err != nil {
return nil, fmt.Errorf("making the output directory for %s: %w", art.Name, err)
}
binary := binaryName(*art)
// One -ldflags, composed: a repeated flag is not a merged one (the build agent's issue 161).
linker := recipeLinkerFlags
if system := strings.TrimSpace(art.System); system != "" {
linker += " -X " + recipeSystemStamp + "=" + system
}
cmd := []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(art.From, "./")}
if r := m.Run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" {
return nil, fmt.Errorf("go build of %s: %s", art.Name, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
}
archive, digest, err := buildstep.Pack(out)
if err != nil {
return nil, fmt.Errorf("packing %s: %w", art.Name, err)
}
artifacts := filepath.Join(m.Work, workspace, "artifacts")
if err := os.MkdirAll(artifacts, 0o700); err != nil {
return nil, err
}
path := filepath.Join(artifacts, strings.ReplaceAll(module, "/", "_")+"."+art.Name+".tar.gz")
if err := os.WriteFile(path, archive, 0o600); err != nil {
return nil, err
}
return BuildResult{Artifact: art.Name, Binary: binary, Digest: digest, Size: len(archive), Path: path}, nil
}
// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed toolchain
// alone, the modules the fetch put in the module's cache and no way to fetch another (GOPROXY=off), so a build never
// reaches the network. A directory that vendors is built from its vendor directory.
func (m *Module) buildEnv(dir string) []string {
mod := "-mod=mod"
if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
mod = "-mod=vendor"
}
return []string{
"CGO_ENABLED=0",
"GOFLAGS=" + recipeFlags + " " + mod,
"GOTOOLCHAIN=local",
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
"GOPROXY=off",
}
}
modules/mesh-build-go/cmd/mesh-build-go/build_test.go
+217
View File
@@ -0,0 +1,217 @@
package main
import (
"archive/tar"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// Issue 1487: the build step, with the compiler faked at the process boundary: it writes the file `go build -o` names.
func manifestWith(artifact string) string {
return `{"module":"demo","version":"1","build":{"artifacts":[` + artifact + `]}}`
}
const goBundle = `{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","binary":"demo-bin","loads":["demo-bin"]}`
// buildModule is a module with a workspace holding a module "modules/demo" with this manifest; the compiler writes
// content to the file go build is told.
func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates, *[]cmdRecord) {
t.Helper()
calls := &[]cmdRecord{}
states := newStates()
m := moduleWith(states, "go1.27.1")
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
*calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")})
for i, a := range args {
if a == "-o" {
if err := os.WriteFile(args[i+1], []byte(content), 0o755); err != nil {
return forgekey.Ran{Status: 1, Stderr: err.Error()}
}
}
}
return forgekey.Ran{}
}
return m, states, calls
}
func runBuild(t *testing.T, m *Module, states *keptStates, artifact string) (BuildResult, delivery.Step) {
t.Helper()
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == "build_go_build" {
if _, err := tool.Run(map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": artifact}); err != nil {
t.Fatal(err)
}
}
}
st := states.waitEnd(t)
var res BuildResult
if st.State == delivery.StepPassed {
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatal(err)
}
}
return res, st
}
// goEntry reads what the build agent's recipe says out of lines 155-191 of mesh-control's internal/builder/toolchain.go
// (testdata/captured/toolchain-go-entry.txt, copied from origin/main 073ccc29): the environment words and flags of the
// compile, the linker flags and the system stamp.
func goEntry(t *testing.T) (compile []string, linker []string, stamp string) {
t.Helper()
raw, err := os.ReadFile(filepath.Join("testdata", "captured", "toolchain-go-entry.txt"))
if err != nil {
t.Fatal(err)
}
text := string(raw)
find := func(pattern string) string {
m := regexp.MustCompile(pattern).FindStringSubmatch(text)
if m == nil {
t.Fatalf("the copied recipe has nothing like %s", pattern)
}
return m[1]
}
for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`Compile: \[\]string\{([^}]*)\}`), -1) {
compile = append(compile, q[1])
}
for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(find(`LinkerFlags: \[\]string\{([^}]*)\}`), -1) {
linker = append(linker, q[1])
}
return compile, linker, find(`SystemStamp: "([^"]+)"`)
}
// 1487/AC1: the build command carries today's flags and environment: the copied recipe says env CGO_ENABLED=0
// GOFLAGS=-trimpath -buildvcs=false, go build, -ldflags -s -w -X main.builtFor=<system>, -o <out>/<binary>, the package.
func TestTheBuildCommandCarriesTodaysFlagsAndEnvironment(t *testing.T) {
m, states, calls := buildModule(t, manifestWith(goBundle), "binary")
res, st := runBuild(t, m, states, "tools")
if st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
compile, linker, stamp := goEntry(t)
// compile is: env CGO_ENABLED=0 "GOFLAGS=-trimpath -buildvcs=false" go build
if len(compile) != 5 || compile[0] != "env" || compile[3] != "go" || compile[4] != "build" {
t.Fatalf("the copied recipe's compile is %v", compile)
}
var c cmdRecord
for _, r := range *calls {
if strings.HasPrefix(r.line, "go ") {
c = r
}
}
env := strings.Join(c.env, "\n")
for _, want := range []string{compile[1], "GOFLAGS=" + strings.TrimPrefix(compile[2], "GOFLAGS=") + " -mod=mod", "GOTOOLCHAIN=local", "GOPROXY=off",
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild")} {
if !strings.Contains(env, want) {
t.Errorf("the environment lacks %q:\n%s", want, env)
}
}
dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
out := filepath.Join(dir, ".mesh-build", "tools")
want := "go -C " + dir + " build -ldflags " + strings.Join(linker, " ") + " -X " + stamp + "=arch -o " + filepath.Join(out, "demo-bin") + " ./cmd/demo-tools"
if c.line != want {
t.Errorf("command = %q\nwant %q", c.line, want)
}
if res.Artifact != "tools" || res.Binary != "demo-bin" || !strings.HasPrefix(res.Digest, "sha256:") || res.Size == 0 {
t.Errorf("result = %+v", res)
}
}
// The archive is the file the digest names, holding one executable of the name binaryName says; two builds of one tree
// are one digest.
func TestTheArchiveHoldsOneExecutableAndTwoBuildsAreOneDigest(t *testing.T) {
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
first, _ := runBuild(t, m, states, "tools")
second, _ := runBuild(t, m, states, "tools")
if first.Digest != second.Digest || first.Path != second.Path {
t.Errorf("two builds: %+v and %+v", first, second)
}
raw, err := os.ReadFile(first.Path)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(raw)
if first.Digest != "sha256:"+hex.EncodeToString(sum[:]) {
t.Errorf("the digest %s is not the archive's", first.Digest)
}
zr, err := gzip.NewReader(strings.NewReader(string(raw)))
if err != nil {
t.Fatal(err)
}
tr := tar.NewReader(zr)
h, err := tr.Next()
if err != nil || h.Name != "demo-bin" || h.Mode != 0o755 {
t.Fatalf("first entry = %+v, %v", h, err)
}
if body, _ := io.ReadAll(tr); string(body) != "the program" {
t.Errorf("the executable holds %q", body)
}
if _, err := tr.Next(); err != io.EOF {
t.Errorf("the archive holds more than one file")
}
}
// 1487/AC3: an artifact of another language, or another kind, fails the step naming it.
func TestAnArtifactOfAnotherLanguageFailsTheStepNamingIt(t *testing.T) {
for artifact, want := range map[string]string{
`{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"],"loads":["index.js"]}`: `language "typescript"`,
`{"name":"tools","kind":"archive","from":"files"}`: "archive",
} {
m, states, calls := buildModule(t, manifestWith(artifact), "x")
_, st := runBuild(t, m, states, "tools")
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, want) || len(*calls) != 0 {
t.Errorf("%s: the step ended %s: %q after %d commands", artifact, st.State, st.Reason, len(*calls))
}
}
m, states, _ := buildModule(t, manifestWith(goBundle), "x")
if _, st := runBuild(t, m, states, "nope"); st.State != delivery.StepFailed || !strings.Contains(st.Reason, `"nope"`) || !strings.Contains(st.Reason, "tools") {
t.Errorf("an unknown artifact: %s %q", st.State, st.Reason)
}
}
// The executable is named as the build agent names it: the artifact's binary, else the package's directory, else the
// artifact.
func TestTheBinaryIsNamedAsTheBuildAgentNamesIt(t *testing.T) {
for artifact, want := range map[string]string{
goBundle: "demo-bin",
`{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/demo-tools","loads":["demo-tools"]}`: "demo-tools",
`{"name":"tools","kind":"bundle","language":"go","system":"arch","from":".","loads":["tools"]}`: "tools",
} {
m, states, _ := buildModule(t, manifestWith(artifact), "x")
if res, st := runBuild(t, m, states, "tools"); res.Binary != want {
t.Errorf("%s: binary %q, want %q (%s %s)", artifact, res.Binary, want, st.State, st.Reason)
}
}
}
func TestAModuleDirectoryOutsideTheWorkspaceIsRefused(t *testing.T) {
m, states, calls := buildModule(t, manifestWith(goBundle), "x")
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == "build_go_build" {
tool.Run(map[string]any{"workspace": "ws1", "module": "../../..", "artifact": "tools"})
}
}
if st := states.waitEnd(t); st.State != delivery.StepFailed || len(*calls) != 0 {
t.Errorf("the step ended %s after %d commands", st.State, len(*calls))
}
}
modules/mesh-build-go/cmd/mesh-build-go/bus.go
+116
View File
@@ -0,0 +1,116 @@
package main
// The integration level's throwaway bus (novox/hq ADR 0470 §2; issue 1486): the module's own program runs a
// nats-server from the Go library, at the one version this module's go.mod names, with JetStream on, its store in the
// run's directory and a loopback port of its own. The program is this binary started again as `mesh-build-go bus
// <dir>`, so the bus is built from the same go.mod as the version the recipe names (ADR 0470 §4) and no second program
// has to be built and shipped; the tools bundle never serves a bus, and a bus never serves tools. It prints its
// address on one line and stops when its standard input closes.
import (
"bufio"
"context"
"fmt"
"io"
"os"
"os/exec"
"os/signal"
"strings"
"syscall"
"time"
"github.com/nats-io/nats-server/v2/server"
)
// busStartBound is how long a bus is given to answer its address.
const busStartBound = 30 * time.Second
// busArg is the first argument that makes this binary a bus.
const busArg = "bus"
// runBus is the bus program: args are the directory of its store. It returns the exit status.
func runBus(args []string) int {
if len(args) != 1 {
fmt.Fprintln(os.Stderr, "usage: mesh-build-go bus <store directory>")
return 2
}
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: args[0], NoLog: true, NoSigs: true})
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
go s.Start()
if !s.ReadyForConnections(busStartBound) {
fmt.Fprintln(os.Stderr, "the bus was not ready within", busStartBound)
s.Shutdown()
return 1
}
fmt.Println(s.ClientURL())
done := make(chan struct{})
go func() { _, _ = io.Copy(io.Discard, os.Stdin); close(done) }()
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT)
select {
case <-done:
case <-sig:
}
s.Shutdown()
s.WaitForShutdown()
return 0
}
// startBus starts the bus of a run as a process of this binary under the step's environment, and answers its address.
func (m *Module) startBus(ctx context.Context, dir string) (string, func(), error) {
if m.StartBus != nil {
return m.StartBus(ctx, dir)
}
exe, err := os.Executable()
if err != nil {
return "", nil, err
}
cmd := exec.Command(exe, busArg, dir)
cmd.Env = baseEnv(m.Home)
in, err := cmd.StdinPipe()
if err != nil {
return "", nil, err
}
out, err := cmd.StdoutPipe()
if err != nil {
return "", nil, err
}
var errb strings.Builder
cmd.Stderr = &errb
if err := cmd.Start(); err != nil {
return "", nil, err
}
stop := func() {
_ = in.Close()
exited := make(chan struct{})
go func() { _ = cmd.Wait(); close(exited) }()
select {
case <-exited:
case <-time.After(10 * time.Second):
_ = cmd.Process.Kill()
<-exited
}
}
line := make(chan string, 1)
go func() {
l, _ := bufio.NewReader(out).ReadString('\n')
line <- strings.TrimSpace(l)
}()
select {
case url := <-line:
if !strings.HasPrefix(url, "nats://") {
stop()
return "", nil, fmt.Errorf("the bus did not answer an address: %s", firstLine(errb.String()+" "+url))
}
return url, stop, nil
case <-time.After(busStartBound + 5*time.Second):
stop()
return "", nil, fmt.Errorf("the bus did not answer an address within %s", busStartBound)
case <-ctx.Done():
stop()
return "", nil, ctx.Err()
}
}
modules/mesh-build-go/cmd/mesh-build-go/digest.go
+278
View File
@@ -0,0 +1,278 @@
package main
// The digest step (novox/hq ADR 0445 §5, ADR 0444; issue 1291): for each package of a workspace that has tests at a
// level, one digest of everything those tests read. The checker keys the verdict it keeps by it, so a package whose
// digest is unchanged is not run again. The tool answers in its call (it is short, ADR 0469 §2).
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
)
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
const DigestVersion = "1"
// goPackage is the part of `go list -json` a digest reads.
type goPackage struct {
ImportPath string
Dir string
ForTest string
Standard bool
DepOnly bool
Deps []string
Module *goModule
// The files a package is built and tested from.
GoFiles, CgoFiles, CFiles, CXXFiles, MFiles, HFiles, FFiles, SFiles, SwigFiles, SwigCXXFiles, SysoFiles []string
TestGoFiles, XTestGoFiles []string
EmbedFiles, TestEmbedFiles, XTestEmbedFiles []string
}
type goModule struct {
Path, Version, Sum string
Main bool
Replace *goModule
Dir string
GoVersion string
}
func (p *goPackage) files() []string {
var all []string
for _, l := range [][]string{p.GoFiles, p.CgoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles,
p.SwigCXXFiles, p.SysoFiles, p.TestGoFiles, p.XTestGoFiles, p.EmbedFiles, p.TestEmbedFiles, p.XTestEmbedFiles} {
all = append(all, l...)
}
sort.Strings(all)
return slicesCompact(all)
}
func slicesCompact(s []string) []string {
out := s[:0:0]
for i, v := range s {
if i == 0 || v != s[i-1] {
out = append(out, v)
}
}
return out
}
// hasTests says whether the package has test files at the level go list was asked for.
func (p *goPackage) hasTests() bool { return len(p.TestGoFiles)+len(p.XTestGoFiles) > 0 }
// DigestResult is what a digest step answers: one digest per package that has tests at the level, and the packages
// that have none.
type DigestResult struct {
Level string `json:"level"`
Packages map[string]string `json:"packages"`
WithoutTests []string `json:"without_tests,omitempty"`
}
func (m *Module) digestTool() StepTool {
return StepTool{
Name: "build_go_digest",
Description: "One digest per package of a fetched workspace, for one level, of everything that package's tests read: " +
"the files of each package the tests reach in the module's own tree, the version and checksum of each module " +
"outside it, the package's test data and embedded files, the Go version, the build tags and, at the " +
"integration level, the postgres major and nats-server versions. Answers in the call.",
Input: map[string]any{
"type": "object",
"properties": map[string]any{
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
},
"required": []string{"workspace", "level"},
},
Run: m.digest,
}
}
func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
workspace, _ := args["workspace"].(string)
level, _ := args["level"].(string)
if !workspaceOK.MatchString(workspace) {
return nil, fmt.Errorf("%q is not a workspace id", workspace)
}
tree := filepath.Join(m.Work, workspace, "tree")
if _, err := os.Stat(tree); err != nil {
return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
}
var tags []string
header := []string{"digest " + DigestVersion, "level " + level, "go " + m.Recipe.Go}
switch level {
case LevelUnit:
case LevelIntegration:
if _, err := m.checkPostgres(ctx); err != nil {
return nil, err
}
tags = []string{"integration"}
header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer)
default:
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
}
header = append(header, "tags "+strings.Join(tags, ","))
dirs, err := m.goModDirs(ctx, tree)
if err != nil {
return nil, err
}
res := DigestResult{Level: level, Packages: map[string]string{}}
for _, d := range dirs {
pkgs, err := m.listPackages(ctx, tree, d, tags)
if err != nil {
return nil, err
}
if err := digestModule(&res, pkgs, tree, header); err != nil {
return nil, err
}
}
sort.Strings(res.WithoutTests)
return res, nil
}
// listPackages is `go list -test -deps -json` of the directory's packages, by import path.
func (m *Module) listPackages(ctx context.Context, tree, dir string, tags []string) (map[string]*goPackage, error) {
abs := filepath.Join(tree, dir)
args := []string{"-C", abs, "list", "-test", "-deps", "-json"}
if len(tags) > 0 {
args = append(args, "-tags", strings.Join(tags, ","))
}
args = append(args, "./...")
r := m.Run(ctx, m.goEnvFor(abs), "go", args...)
if r.Status != 0 || r.Err != "" {
return nil, fmt.Errorf("go list in %s: %s", dir, firstLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
}
return parsePackages(r.Stdout)
}
// parsePackages reads `go list -json`'s stream of objects.
func parsePackages(out string) (map[string]*goPackage, error) {
pkgs := map[string]*goPackage{}
dec := json.NewDecoder(strings.NewReader(out))
for {
var p goPackage
if err := dec.Decode(&p); err == io.EOF {
return pkgs, nil
} else if err != nil {
return nil, fmt.Errorf("go list answered something that is not its JSON: %w", err)
}
pkgs[p.ImportPath] = &p
}
}
// digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main
// module that is listed, not only a dependency, and not a variant of another.
func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header []string) error {
roots := []string{}
for path, p := range pkgs {
if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main {
roots = append(roots, path)
}
}
sort.Strings(roots)
for _, path := range roots {
root := pkgs[path]
if !root.hasTests() {
res.WithoutTests = append(res.WithoutTests, path)
continue
}
testMain := pkgs[path+".test"]
if testMain == nil {
return fmt.Errorf("go list gave no test binary for %s", path)
}
d, err := packageDigest(root, testMain, pkgs, tree, header)
if err != nil {
return fmt.Errorf("%s: %w", path, err)
}
res.Packages[path] = d
}
return nil
}
// packageDigest is the digest of one package: a hash over a sorted text of the header, the files of every package its
// tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of
// every module outside it, and the package's own test data. A package of the standard library is the Go version's.
func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header []string) (string, error) {
reached := map[string]*goPackage{root.ImportPath: root}
for _, dep := range testMain.Deps {
if p := pkgs[dep]; p != nil {
reached[dep] = p
}
}
lines := []string{}
for _, p := range reached {
if p.Standard || p.Module == nil {
continue
}
mod := p.Module
if mod.Replace != nil {
mod = mod.Replace
}
if !mod.Main && mod.Version != "" {
lines = append(lines, "module "+mod.Path+"@"+mod.Version+" "+mod.Sum)
continue
}
for _, f := range p.files() {
l, err := fileLine(tree, p, f)
if err != nil {
return "", err
}
lines = append(lines, l)
}
}
data := filepath.Join(root.Dir, "testdata")
if _, err := os.Stat(data); err == nil {
err := filepath.WalkDir(data, func(path string, e fs.DirEntry, err error) error {
if err != nil || e.IsDir() {
return err
}
l, err := contentLine(tree, root, path)
lines = append(lines, l)
return err
})
if err != nil {
return "", err
}
}
sort.Strings(lines)
h := sha256.New()
for _, l := range append(append([]string{}, header...), lines...) {
h.Write([]byte(l + "\n"))
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func fileLine(tree string, p *goPackage, name string) (string, error) {
return contentLine(tree, p, filepath.Join(p.Dir, name))
}
// contentLine is "file <path> <sha256>": the path is the file's in the tree (the import path and the name when it lies
// outside, as a local replacement does), so a digest does not depend on where the workspace is.
func contentLine(tree string, p *goPackage, path string) (string, error) {
label := p.ImportPath + "/" + filepath.Base(path)
if rel, err := filepath.Rel(tree, path); err == nil && !strings.HasPrefix(rel, "..") {
label = filepath.ToSlash(rel)
}
info, err := os.Lstat(path)
if err != nil {
return "", err
}
if info.Mode()&os.ModeSymlink != 0 {
to, _ := os.Readlink(path)
return "link " + label + " " + to, nil
}
raw, err := os.ReadFile(path)
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return "file " + label + " " + hex.EncodeToString(sum[:]), nil
}
modules/mesh-build-go/cmd/mesh-build-go/digest_test.go
+195
View File
@@ -0,0 +1,195 @@
package main
import (
"context"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// Issue 1291: the digests of a fixture module (testdata/repos/digest): a's tests import b and read a/testdata, c's
// tests read an embedded file, d is reached by no test. They run the real `go list` (it loads packages, it builds
// none), through the step runner; only the listing of the tree's go.mod files and postgres's version are answered.
// digestModule is a module over a copy of the fixture, in a workspace of its own.
func digestFixture(t *testing.T) (*Module, string) {
t.Helper()
work := t.TempDir()
tree := filepath.Join(work, "ws1", "tree")
if err := os.MkdirAll(tree, 0o755); err != nil {
t.Fatal(err)
}
if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", "digest")+"/.", tree).CombinedOutput(); err != nil {
t.Fatalf("%v %s", err, out)
}
m := moduleWith(newStates(), "go1.27.1")
m.Home, m.Work, m.Cache = t.TempDir(), work, t.TempDir()
real := StepRunner(m.Home)
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
switch name {
case "git":
return forgekey.Ran{Stdout: "go.mod\n"}
case "postgres":
return forgekey.Ran{Stdout: "postgres (PostgreSQL) " + strconv.Itoa(m.Recipe.PostgresMajor) + ".1\n"}
}
return real(ctx, env, name, args...)
}
return m, tree
}
func digests(t *testing.T, m *Module, level string) DigestResult {
t.Helper()
out, err := m.digest(context.Background(), map[string]any{"workspace": "ws1", "level": level})
if err != nil {
t.Fatal(err)
}
return out.(DigestResult)
}
func edit(t *testing.T, tree, name, text string) {
t.Helper()
if err := os.WriteFile(filepath.Join(tree, name), []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
// 1291/AC1: a file of b changes: a's digest changes (its tests import b), c's does not.
func TestADependencysFileChangesTheDigestOfThePackageWhoseTestsImportIt(t *testing.T) {
t.Parallel()
m, tree := digestFixture(t)
before := digests(t, m, "unit")
for _, p := range []string{"fixture/digest/a", "fixture/digest/c"} {
if len(before.Packages[p]) != 64 {
t.Fatalf("no digest for %s: %+v", p, before)
}
}
edit(t, tree, "b/b.go", "package b\n\n// B is reached by a's tests.\nfunc B() int { return 2 + 0 }\n")
after := digests(t, m, "unit")
if after.Packages["fixture/digest/a"] == before.Packages["fixture/digest/a"] {
t.Errorf("a's digest did not change when b did")
}
if after.Packages["fixture/digest/c"] != before.Packages["fixture/digest/c"] {
t.Errorf("c's digest changed when b did")
}
if again := digests(t, m, "unit"); again.Packages["fixture/digest/a"] != after.Packages["fixture/digest/a"] {
t.Errorf("one tree has two digests")
}
}
// 1291/AC2: a file under a's testdata/ changes: a's digest changes. An embedded file does the same for c.
func TestATestdataFileAndAnEmbeddedFileChangeTheDigest(t *testing.T) {
t.Parallel()
m, tree := digestFixture(t)
before := digests(t, m, "unit")
edit(t, tree, "a/testdata/x.txt", "y\n")
after := digests(t, m, "unit")
if after.Packages["fixture/digest/a"] == before.Packages["fixture/digest/a"] {
t.Errorf("a's digest did not change with its testdata")
}
if after.Packages["fixture/digest/c"] != before.Packages["fixture/digest/c"] {
t.Errorf("c's digest changed with a's testdata")
}
edit(t, tree, "c/note.txt", "another note\n")
if again := digests(t, m, "unit"); again.Packages["fixture/digest/c"] == after.Packages["fixture/digest/c"] {
t.Errorf("c's digest did not change with its embedded file")
}
}
// 1291/AC3: one tree at the unit and the integration level: the digests differ; at the integration level a change of the
// recipe's postgres major (or of the nats-server version) changes it, at the unit level it does not.
func TestTheLevelTheStoreAndTheBusAreInTheDigest(t *testing.T) {
t.Parallel()
m, _ := digestFixture(t)
unit, integration := digests(t, m, "unit"), digests(t, m, "integration")
a := "fixture/digest/a"
if unit.Packages[a] == integration.Packages[a] {
t.Errorf("the unit and the integration digest of a are one")
}
m.Recipe.PostgresMajor++
if got := digests(t, m, "integration"); got.Packages[a] == integration.Packages[a] {
t.Errorf("a new postgres major left the integration digest as it was")
}
if got := digests(t, m, "unit"); got.Packages[a] != unit.Packages[a] {
t.Errorf("a new postgres major moved the unit digest")
}
m.Recipe.PostgresMajor--
m.Recipe.NatsServer = "9.9.9"
if got := digests(t, m, "integration"); got.Packages[a] == integration.Packages[a] {
t.Errorf("a new nats-server version left the integration digest as it was")
}
m.Recipe.NatsServer = "2.11.17"
m.Recipe.Go = "go1.27.2"
if got := digests(t, m, "unit"); got.Packages[a] == unit.Packages[a] {
t.Errorf("a new Go version left the unit digest as it was")
}
}
// 1291/AC4: a change in a package no test of a reaches leaves a's digest as it was; d, with no tests, has none.
func TestAPackageNoTestReachesChangesNoDigest(t *testing.T) {
t.Parallel()
m, tree := digestFixture(t)
before := digests(t, m, "unit")
edit(t, tree, "d/d.go", "package d\n\n// D is reached by no test.\nfunc D() int { return 5 }\n")
after := digests(t, m, "unit")
for p, d := range before.Packages {
if after.Packages[p] != d {
t.Errorf("the digest of %s changed with a package no test of it reaches", p)
}
}
if _, has := after.Packages["fixture/digest/d"]; has || strings.Join(after.WithoutTests, " ") != "fixture/digest/b fixture/digest/d" {
t.Errorf("without tests = %v, packages = %v", after.WithoutTests, after.Packages)
}
}
// The digest does not depend on where the workspace lies.
func TestTheDigestIsTheSameInAnotherWorkspace(t *testing.T) {
t.Parallel()
m1, _ := digestFixture(t)
m2, _ := digestFixture(t)
if d1, d2 := digests(t, m1, "unit"), digests(t, m2, "unit"); d1.Packages["fixture/digest/a"] != d2.Packages["fixture/digest/a"] {
t.Errorf("one tree in two places has two digests")
}
}
// A module outside the tree counts by its version and checksum, not by its files: the package of mesh-sdk, as go list
// gave it for this module's own dependency (testdata/captured/golist-external.json, which README describes).
func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("testdata", "captured", "golist-external.json"))
if err != nil {
t.Fatal(err)
}
pkgs, err := parsePackages(string(raw))
if err != nil {
t.Fatal(err)
}
ext := pkgs["git.novox.be/novox/mesh-sdk/go/buildstep"]
if ext == nil || ext.Module == nil || ext.Module.Sum == "" || ext.Module.Version == "" || ext.Module.Main {
t.Fatalf("the capture holds no external package with a version and a checksum: %+v", ext)
}
dir := t.TempDir()
os.WriteFile(filepath.Join(dir, "r.go"), []byte("package r\n"), 0o644)
root := &goPackage{ImportPath: "x/r", Dir: dir, Module: &goModule{Path: "x", Main: true}, GoFiles: []string{"r.go"}, TestGoFiles: []string{"r.go"}}
testMain := &goPackage{ImportPath: "x/r.test", Deps: []string{ext.ImportPath}}
pkgs[root.ImportPath] = root
digest := func() string {
d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"})
if err != nil {
t.Fatal(err)
}
return d
}
d1 := digest()
ext.Module.Sum = "h1:changed="
if d2 := digest(); d1 == d2 {
t.Errorf("a new checksum of a module outside the tree left the digest as it was")
}
ext.Module.Sum, ext.Module.Version = "h1:changed=", "v9.9.9"
if d3 := digest(); d3 == d1 {
t.Errorf("a new version left the digest as it was")
}
}
modules/mesh-build-go/cmd/mesh-build-go/emit_test.go
+55
View File
@@ -0,0 +1,55 @@
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"slices"
"testing"
)
// Issues 1483 and 1488 (for the indexer, issue 1444): the guard emits toolchain.reported after each write of the
// toolchains key, with the node and the language.
func TestTheGuardEmitsToolchainReportedAfterEachWrite(t *testing.T) {
m := moduleWith(newStates(), "go1.27.1")
type event struct {
typ string
body map[string]string
}
var got []event
m.Emit = func(typ string, body any) error {
raw, _ := json.Marshal(body)
var b map[string]string
json.Unmarshal(raw, &b)
got = append(got, event{typ, b})
return nil
}
if err := m.Check(context.Background()); err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].typ != "toolchain.reported" || got[0].body["node"] != "ace" || got[0].body["language"] != "go" || len(got[0].body) != 2 {
t.Errorf("events = %+v, want one toolchain.reported for ace, go", got)
}
m.Check(context.Background())
if len(got) != 2 {
t.Errorf("a second guard pass emitted %d events in all, want 2", len(got))
}
}
// The event is carried by the module's stream, not named in emits (ADR 0363: one place); the validator checks that.
func TestTheManifestDeclaresTheToolchainsStream(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m struct {
Streams []struct {
Name string `json:"name"`
Events []string `json:"events"`
} `json:"streams"`
}
if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) != 1 || m.Streams[0].Name != "toolchains-go" || !slices.Equal(m.Streams[0].Events, []string{"toolchain.**"}) {
t.Errorf("streams = %+v (%v)", m.Streams, err)
}
}
modules/mesh-build-go/cmd/mesh-build-go/fetch.go
+174
View File
@@ -0,0 +1,174 @@
package main
// The fetch step (novox/hq ADR 0405 §5 §6, ADR 0466 §4 §5, ADR 0469 §2; issue 1485): a commit of a repository of
// the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and the Go modules every
// go.mod in it names, downloaded into the module's cache and verified against go.sum.
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
const ForgeAlias = "mesh-forge"
// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
// fetches and every round it checks its toolchain. A workspace lies in the module's work directory, never in its
// home (ADR 0466 §1).
const WorkspaceAge = 24 * time.Hour
var (
repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`)
)
// FetchResult is what a passed fetch step says: the workspace it made, the commit in it and the directories whose
// Go modules it downloaded.
type FetchResult struct {
Workspace string `json:"workspace"`
Commit string `json:"commit"`
Modules []string `json:"modules"`
}
// fetchTool is the fetch step tool.
func (m *Module) fetchTool() StepTool {
return StepTool{
Name: "build_go_fetch",
Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
"workspace of its own, and download and verify the Go modules every go.mod in it names. Answers a step id at " +
"once; the step ends on its key in the steps state with the workspace's id.",
Input: map[string]any{
"type": "object",
"properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
},
"required": []string{"repository", "commit"},
},
Long: true,
Run: m.fetch,
}
}
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
repository, _ := args["repository"].(string)
commit, _ := args["commit"].(string)
ref, _ := args["ref"].(string)
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
}
m.CleanWorkspaces(m.now())
id := buildstep.NewID()
root := filepath.Join(m.Work, id)
tree := filepath.Join(root, "tree")
if err := os.MkdirAll(tree, 0o700); err != nil {
return nil, err
}
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref); err != nil {
return nil, err
}
dirs, err := m.goModDirs(ctx, tree)
if err != nil {
return nil, err
}
for _, d := range dirs {
if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil {
continue // a module that vendors resolves nothing
}
for _, verb := range []string{"download", "verify"} {
r := m.Run(ctx, m.goEnv(), "go", "-C", filepath.Join(tree, d), "mod", verb)
if r.Status != 0 || r.Err != "" {
return nil, fmt.Errorf("go mod %s in %s: %s", verb, d, firstLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr))
}
}
}
return FetchResult{Workspace: id, Commit: commit, Modules: dirs}, nil
}
// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order.
func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod")
if r.Status != 0 || r.Err != "" {
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
}
dirs := []string{}
for _, l := range strings.Split(r.Stdout, "\n") {
if l = strings.TrimSpace(l); l != "" {
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
}
}
sort.Strings(dirs)
return dirs, nil
}
// goEnv is what a go command of a step is given beyond the base environment: the installed toolchain and no other,
// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by
// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
// that vendors, see goEnvFor.
func (m *Module) goEnv() []string {
return []string{
"GOTOOLCHAIN=local",
"GOFLAGS=-mod=mod",
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
"GOPRIVATE=git.novox.be",
"GIT_TERMINAL_PROMPT=0",
"GIT_SSH_COMMAND=ssh -o BatchMode=yes",
}
}
// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor
// directory, as fetch downloaded nothing for it, and every other by its go.mod and go.sum.
func (m *Module) goEnvFor(dir string) []string {
env := m.goEnv()
if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
env = append(env, "GOFLAGS=-mod=vendor") // the last value of a name is the one used
}
return env
}
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
func (m *Module) CleanWorkspaces(now time.Time) {
entries, err := os.ReadDir(m.Work)
if err != nil {
return
}
for _, e := range entries {
info, err := e.Info()
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
continue
}
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
}
}
}
// clock is the time now: Now when a test set it, else the wall clock.
func (m *Module) clock() time.Time {
if m.Now != nil {
return m.Now()
}
return time.Now()
}
func (m *Module) now() time.Time { return m.clock() }
func firstLine(s string) string {
for _, l := range strings.Split(s, "\n") {
if l = strings.TrimSpace(l); l != "" {
return l
}
}
return ""
}
modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go
+292
View File
@@ -0,0 +1,292 @@
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const commitSHA = "0123456789abcdef0123456789abcdef01234567"
// cmdRecord is one command a step ran.
type cmdRecord struct {
env []string
line string
}
// fakeStep answers the commands of a fetch: git succeeds, rev-parse answers the commit, ls-files answers the
// go.mod files, and a go command answers by rule.
type fakeStep struct {
mu sync.Mutex
calls []cmdRecord
gomods string
gate chan struct{} // when set, the first git command waits for it
goRule func(line string) forgekey.Ran
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
}
func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
line := name + " " + strings.Join(args, " ")
f.mu.Lock()
f.calls = append(f.calls, cmdRecord{env: env, line: line})
f.mu.Unlock()
if f.gate != nil && strings.Contains(line, " init ") {
<-f.gate
}
switch {
case name == "git" && strings.Contains(line, " rev-parse HEAD"):
return forgekey.Ran{Stdout: commitSHA + "\n"}
case name == "git" && strings.Contains(line, " ls-files "):
return forgekey.Ran{Stdout: f.gomods}
case name == "git" && strings.Contains(line, " checkout "):
for i, a := range args {
if a == "-C" {
for _, v := range f.vendors {
os.MkdirAll(filepath.Join(args[i+1], v, "vendor"), 0o755)
os.WriteFile(filepath.Join(args[i+1], v, "vendor", "modules.txt"), nil, 0o644)
}
}
}
case name == "go" && f.goRule != nil:
return f.goRule(line)
}
return forgekey.Ran{}
}
func (f *fakeStep) lines() []string {
f.mu.Lock()
defer f.mu.Unlock()
out := []string{}
for _, c := range f.calls {
out = append(out, c.line)
}
return out
}
func fetchModule(t *testing.T, f *fakeStep) (*Module, *keptStates) {
t.Helper()
states := newStates()
m := moduleWith(states, "go1.27.1")
m.Run, m.Home, m.Work, m.Cache = f.run, t.TempDir(), t.TempDir(), t.TempDir()
return m, states
}
func fetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
t.Helper()
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == "build_go_fetch" {
return tool.Run
}
}
t.Fatal("no tool build_go_fetch")
return nil
}
var fetchArgs = map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}
// 1485/AC1: git is asked through FetchCommit with the ssh remote, and go mod download and verify run in each go.mod
// directory (not in one that vendors) with GOTOOLCHAIN=local, GOFLAGS=-mod=mod and the module's cache paths.
func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) {
f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}}
m, states := fetchModule(t, f)
if _, err := fetchTool(t, m)(fetchArgs); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
if st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
lines := strings.Join(f.lines(), "\n")
for _, want := range []string{
"remote add origin ssh://mesh-forge/novox/mesh-sdk.git",
"fetch --no-tags --depth 1 origin " + commitSHA,
"checkout -q --detach " + commitSHA,
} {
if !strings.Contains(lines, want) {
t.Errorf("git was not asked %q:\n%s", want, lines)
}
}
downloads, verifies := 0, 0
for _, c := range f.calls {
switch {
case strings.HasPrefix(c.line, "go -C ") && strings.HasSuffix(c.line, " mod download"):
downloads++
case strings.HasPrefix(c.line, "go -C ") && strings.HasSuffix(c.line, " mod verify"):
verifies++
default:
continue
}
if strings.Contains(c.line, "vend") {
t.Errorf("a module that vendors was downloaded: %s", c.line)
}
env := strings.Join(c.env, " ")
for _, want := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod"), "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), "GOPRIVATE=git.novox.be"} {
if !strings.Contains(env, want) {
t.Errorf("%s: environment lacks %s: %s", c.line, want, env)
}
}
}
if downloads != 2 || verifies != 2 {
t.Errorf("%d downloads and %d verifies, want 2 each (go.mod and sub/dir/go.mod):\n%s", downloads, verifies, lines)
}
var res FetchResult
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatal(err)
}
if res.Commit != commitSHA || strings.Join(res.Modules, ",") != ".,sub/dir,vend" || res.Workspace == "" {
t.Errorf("result = %s", st.Result)
}
if _, err := os.Stat(filepath.Join(m.Work, res.Workspace, "tree")); err != nil {
t.Errorf("the workspace is not in the work directory: %v", err)
}
}
// 1485/AC2: a variable such as MESH_REGISTRY_PASSWORD or a *_TOKEN of the process is seen by no command of the step:
// the runner builds the environment from scratch, and the step passes it nothing of the kind.
func TestNoCommandOfTheStepSeesAPasswordOrAToken(t *testing.T) {
t.Setenv("MESH_REGISTRY_PASSWORD", "hunter2-registry")
t.Setenv("FORGE_TOKEN", "hunter2-token")
t.Setenv("HOME", "/not/the/account")
r := StepRunner("/home/account")
got := r(context.Background(), []string{"EXTRA=1"}, "env")
if got.Status != 0 {
t.Fatalf("env: %+v", got)
}
for _, secret := range []string{"hunter2", "MESH_REGISTRY_PASSWORD", "FORGE_TOKEN", "/not/the/account"} {
if strings.Contains(got.Stdout, secret) {
t.Errorf("a command saw %q:\n%s", secret, got.Stdout)
}
}
for _, want := range []string{"HOME=/home/account", "EXTRA=1", "PATH="} {
if !strings.Contains(got.Stdout, want) {
t.Errorf("a command lacks %q:\n%s", want, got.Stdout)
}
}
f := &fakeStep{gomods: "go.mod\n"}
m, states := fetchModule(t, f)
fetchTool(t, m)(fetchArgs)
states.waitEnd(t)
for _, c := range f.calls {
for _, w := range c.env {
if k, _, _ := strings.Cut(w, "="); strings.HasSuffix(k, "TOKEN") || strings.HasSuffix(k, "PASSWORD") || strings.HasSuffix(k, "SECRET") {
t.Errorf("%s was given %s", c.line, w)
}
}
}
}
// 1485/AC3: go mod verify failing ends the step failed naming the directory and the first line of its output.
func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) {
f := &fakeStep{gomods: "go.mod\ntools/go.mod\n", goRule: func(line string) forgekey.Ran {
if strings.HasSuffix(line, "tools mod verify") || strings.Contains(line, "/tools mod verify") {
return forgekey.Ran{Status: 1, Stdout: "\nexample.org/x v1.0.0: dir has been modified\nsecond line\n"}
}
return forgekey.Ran{}
}}
m, states := fetchModule(t, f)
if _, err := fetchTool(t, m)(fetchArgs); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
if st.State != delivery.StepFailed {
t.Fatalf("the step ended %s, want failed", st.State)
}
if !strings.Contains(st.Reason, "tools") || !strings.Contains(st.Reason, "example.org/x v1.0.0: dir has been modified") || strings.Contains(st.Reason, "second line") {
t.Errorf("reason = %q", st.Reason)
}
}
// 1485/AC4: the call answers a step id before the fetch ends, and the key then ends passed with the workspace.
func TestTheCallAnswersAStepIdBeforeTheFetchEnds(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n", gate: make(chan struct{})}
m, states := fetchModule(t, f)
out, err := fetchTool(t, m)(fetchArgs)
if err != nil {
t.Fatal(err)
}
answer := out.(map[string]any)
id, _ := answer["step"].(string)
if id == "" || answer["key"] != "ace."+id {
t.Fatalf("answer = %v, want a step id and its key", answer)
}
select {
case <-states.ended:
t.Fatal("the step ended while git was held")
case <-time.After(50 * time.Millisecond):
}
states.mu.Lock()
running := states.steps["ace."+id]
states.mu.Unlock()
if len(running) != 1 || running[0].State != delivery.StepRunning {
t.Errorf("the key holds %+v, want one running step", running)
}
close(f.gate)
select {
case key := <-states.ended:
if key != "ace."+id {
t.Errorf("ended %s", key)
}
case <-time.After(3 * time.Second):
t.Fatal("the step never ended")
}
states.mu.Lock()
last := states.steps["ace."+id][len(states.steps["ace."+id])-1]
states.mu.Unlock()
var res FetchResult
if last.State != delivery.StepPassed || json.Unmarshal(last.Result, &res) != nil || res.Workspace == "" {
t.Errorf("the key ends %+v", last)
}
}
func TestARepositoryThatIsNotOwnerSlashNameIsRefusedBeforeAnythingRuns(t *testing.T) {
for _, repo := range []string{"", "mesh-sdk", "../etc/passwd", "a/b/c", "a/..", "-o/x", "https://x/y"} {
f := &fakeStep{}
m, states := fetchModule(t, f)
fetchTool(t, m)(map[string]any{"repository": repo, "commit": commitSHA})
st := states.waitEnd(t)
if st.State != delivery.StepFailed || len(f.lines()) != 0 {
t.Errorf("%q: the step ended %s after %d commands", repo, st.State, len(f.lines()))
}
}
}
// Workspaces older than a day are removed; younger ones, and anything that is not a workspace, stay.
func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) {
m := &Module{Work: t.TempDir()}
for name, age := range map[string]time.Duration{"old1": 25 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} {
dir := filepath.Join(m.Work, name)
os.MkdirAll(filepath.Join(dir, "tree"), 0o755)
when := time.Now().Add(-age)
os.Chtimes(dir, when, when)
}
m.CleanWorkspaces(time.Now())
for name, want := range map[string]bool{"old1": false, "young": true, "Keep-This": true} {
_, err := os.Stat(filepath.Join(m.Work, name))
if (err == nil) != want {
t.Errorf("%s present = %v, want %v", name, err == nil, want)
}
}
}
// The work and cache directories the code uses are the ones the manifest places, and neither is in the home.
func TestWorkAndCacheAreTheDirectoriesTheManifestPlacesOutsideTheHome(t *testing.T) {
m := readManifest(t)
home := resource(t, m, "account")["home"].(string)
for id, path := range map[string]string{"work": WorkDir, "cache": CacheDir} {
if resource(t, m, id)["path"] != path {
t.Errorf("%s: the manifest places %v, the code uses %s", id, resource(t, m, id)["path"], path)
}
if path == home || strings.HasPrefix(path, home+"/") {
t.Errorf("%s %s is in the home %s", id, path, home)
}
}
}
modules/mesh-build-go/cmd/mesh-build-go/forge_test.go
+246
View File
@@ -0,0 +1,246 @@
package main
// The forge side of a build module (novox/hq ADR 0466 §2 and §4, issue 1484). This file is the same for every build
// module: mesh-build-node takes a copy and changes the three constants below.
import (
"context"
"encoding/base64"
"encoding/binary"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const (
forgeModule = "mesh-build-go"
forgeHome = "/var/lib/mesh-build-go"
forgeModuleD = "../.." // the module's directory, where module.json and files/ are
)
type forgeManifest struct {
Requires []any `json:"requires"`
State []any `json:"state"`
Resources []map[string]any `json:"resources"`
Build struct {
Artifacts []map[string]any `json:"artifacts"`
} `json:"build"`
}
func readForgeManifest(t *testing.T) forgeManifest {
t.Helper()
raw, err := os.ReadFile(filepath.Join(forgeModuleD, "module.json"))
if err != nil {
t.Fatal(err)
}
var m forgeManifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
return m
}
func forgeTemplate(t *testing.T, r map[string]any) string {
t.Helper()
name, _ := r["template"].(string)
if name == "" {
t.Fatalf("resource %v names no template", r["id"])
}
raw, err := os.ReadFile(filepath.Join(forgeModuleD, filepath.FromSlash(name)))
if err != nil {
t.Fatalf("resource %v names %s: %v", r["id"], name, err)
}
return string(raw)
}
func forgeResource(t *testing.T, m forgeManifest, id string) map[string]any {
t.Helper()
for _, r := range m.Resources {
if r["id"] == id {
if r["owner"] != forgeModule {
t.Errorf("%s is owned by %v, not the module's own account", id, r["owner"])
}
if p, _ := r["path"].(string); !strings.HasPrefix(p, "${dir:") || strings.Contains(p, "account-home") {
t.Errorf("%s is at %q: a path of the account's own home (${dir:...}), never the operator's", id, p)
}
return r
}
}
t.Fatalf("no resource %q", id)
return nil
}
var forgeSample = map[string]string{
"known_host": "[novox.internal]:222 ssh-ed25519 AAAAC3Nza",
"host_key": "ssh-ed25519 AAAAC3Nza",
"ssh_host": "novox.internal",
"ssh_alias": "mesh-forge",
"ssh_port": "222",
"ssh_base": "ssh://git@novox.internal:222/",
"https_base": "https://git.novox.be/",
}
// forgeRender fills ${bound:forge-account:<key>} with the sample values, as the provisioner client does.
func forgeRender(t *testing.T, s string) string {
t.Helper()
for k, v := range forgeSample {
s = strings.ReplaceAll(s, "${bound:forge-account:"+k+"}", v)
}
if strings.Contains(s, "${") {
t.Errorf("unfilled placeholder in %q", s)
}
return s
}
// 1484/AC1: given sample bound values, the ssh config has Host mesh-forge, IdentitiesOnly yes and the key under the
// module's own home; known_hosts carries the host key; git's config rewrites https_base to ssh://mesh-forge/; no
// ${ is left.
func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) {
m := readForgeManifest(t)
cfg := forgeResource(t, m, "ssh-config")
want := "Host mesh-forge\n HostName novox.internal\n Port 222\n HostKeyAlias mesh-forge\n User git\n IdentityFile ~/.ssh/mesh_ed25519\n IdentitiesOnly yes\n"
if got := forgeRender(t, forgeTemplate(t, cfg)); got != want {
t.Errorf("ssh config = %q, want %q", got, want)
}
if cfg["path"] != "${dir:ssh-dir}/config" || cfg["mode"] != "0600" || cfg["into"] != nil {
t.Errorf("ssh config resource = %v, want the whole file ${dir:ssh-dir}/config at mode 0600", cfg)
}
kh := forgeResource(t, m, "known-hosts")
if got := strings.TrimSpace(forgeRender(t, forgeTemplate(t, kh))); got != "mesh-forge ssh-ed25519 AAAAC3Nza" {
t.Errorf("known_hosts = %q", got)
}
g := forgeResource(t, m, "git-config")
wantGit := "[url \"ssh://mesh-forge/\"]\n\tinsteadOf = https://git.novox.be/\n"
if got := forgeRender(t, forgeTemplate(t, g)); got != wantGit {
t.Errorf("git config = %q, want %q", got, wantGit)
}
dir := forgeResource(t, m, "ssh-dir")
if dir["mode"] != "0700" || dir["path"] != "${dir:state}/.ssh" {
t.Errorf("ssh-dir = %v, want ${dir:state}/.ssh at mode 0700", dir)
}
}
// 1484/AC2: the module requires forge-account, keeps the state keys, and every file resource takes its text from a
// template in the files/ archive (novox/hq ADR 0349 §2).
func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) {
m := readForgeManifest(t)
if len(m.Requires) != 1 || m.Requires[0] != "forge-account" {
t.Errorf("requires = %v, want [forge-account]", m.Requires)
}
keeps := false
for _, s := range m.State {
if s == "keys" {
keeps = true
}
}
if !keeps {
t.Errorf("state = %v lacks keys", m.State)
}
files := 0
for _, r := range m.Resources {
if _, inline := r["content"]; inline {
t.Errorf("resource %v keeps its text inline", r["id"])
}
if r["type"] == "file" {
files++
forgeTemplate(t, r)
}
}
if files == 0 {
t.Errorf("no file resource: the forge's ssh files are missing")
}
packs := 0
for _, a := range m.Build.Artifacts {
if a["kind"] == "archive" && a["from"] == "files" && a["name"] == "files" {
packs++
}
}
if packs != 1 {
t.Errorf("the build packs files/ into %d archives named files, not one", packs)
}
}
// forgeEd25519Line is a public key line with a well-formed ed25519 blob.
func forgeEd25519Line(comment string) string {
blob := []byte{}
for _, part := range [][]byte{[]byte("ssh-ed25519"), make([]byte, 32)} {
l := make([]byte, 4)
binary.BigEndian.PutUint32(l, uint32(len(part)))
blob = append(append(blob, l...), part...)
}
return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(blob) + " " + comment
}
type forgeKeys struct{ puts map[string][]byte }
func (f *forgeKeys) Put(key string, value any) (uint64, error) {
raw, _ := json.Marshal(value)
if f.puts == nil {
f.puts = map[string][]byte{}
}
f.puts[key] = raw
return uint64(len(f.puts)), nil
}
// forgeKeygen stands in for ssh-keygen: it writes the pair the command names.
func forgeKeygen(_ context.Context, _ []string, name string, args ...string) forgekey.Ran {
var file, comment string
for i, a := range args {
switch a {
case "-f":
file = args[i+1]
case "-C":
comment = args[i+1]
}
}
if name != "ssh-keygen" || file == "" {
return forgekey.Ran{Status: 1, Stderr: "unexpected"}
}
os.WriteFile(file, []byte("PRIVATE\n"), 0o600)
os.WriteFile(file+".pub", []byte(forgeEd25519Line(comment)+"\n"), 0o644)
return forgekey.Ran{}
}
// 1484/AC3: given the bundle started with an empty home, the keeper writes <node>.<module> with kind builder, and
// the key is in the home's .ssh.
func TestTheKeeperWritesTheModulesEntryWithKindBuilder(t *testing.T) {
home, st := t.TempDir(), &forgeKeys{}
if err := keeperFor(home, "ace", forgeModule, forgeKeygen, st).Once(context.Background()); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(home, ".ssh", "mesh_ed25519")); err != nil {
t.Errorf("no key in an empty home: %v", err)
}
raw, ok := st.puts["ace."+forgeModule]
if !ok || len(st.puts) != 1 {
t.Fatalf("entries = %v, want one at ace.%s", st.puts, forgeModule)
}
var e forgekey.KeyEntry
if err := json.Unmarshal(raw, &e); err != nil {
t.Fatal(err)
}
if e.Kind != "builder" || e.Node != "ace" || e.Account != forgeModule || !strings.HasPrefix(e.Public, "ssh-ed25519 ") {
t.Errorf("entry = %s", raw)
}
if strings.Contains(strings.ToUpper(string(raw)), "PRIVATE") {
t.Errorf("the entry carries private material: %s", raw)
}
}
// The home the manifest places is the one the account is given, so the keeper's home and the templates' agree.
func TestTheAccountsHomeIsTheOneTheManifestNames(t *testing.T) {
m := readForgeManifest(t)
for _, r := range m.Resources {
if r["type"] == "user" && r["name"] == forgeModule && r["home"] != forgeHome {
t.Errorf("the account's home is %v, want %s", r["home"], forgeHome)
}
}
}
modules/mesh-build-go/cmd/mesh-build-go/integration.go
+77
View File
@@ -0,0 +1,77 @@
package main
// The integration level (novox/hq ADR 0445 §2 §3, ADR 0470; issue 1486): `go test -tags integration` over the
// workspace, against a throwaway postgres and a throwaway bus started for this run alone, handed to the tests as
// MESH_TEST_POSTGRES and MESH_TEST_NATS. Its setup (compiling, the store, the bus) and its running time are measured and
// judged apart, 30 s and 60 s; both servers are stopped and the run's directory removed whatever the tests did.
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
// The integration level's bounds (ADR 0445 §3).
const (
IntegrationSetup = 30 * time.Second
IntegrationRun = 60 * time.Second
)
func (m *Module) integrationBounds() Bounds {
b := m.IntegrationBounds
if b.Setup <= 0 {
b.Setup = IntegrationSetup
}
if b.Run <= 0 {
b.Run = IntegrationRun
}
return b
}
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string) (res *TestResult, err error) {
major, err := m.checkPostgres(ctx)
if err != nil {
return nil, err // a *buildstep.Refusal: this level only
}
run := filepath.Join(m.Work, workspace, "run-"+buildstep.NewID())
if err := os.MkdirAll(run, 0o700); err != nil {
return nil, err
}
// The run's directory goes, and the servers stop, however the level ends: passed, failed, a panic.
defer os.RemoveAll(run)
began := m.clock()
dirs, err := m.goModDirs(ctx, tree)
if err != nil {
return nil, err
}
postgres, stopStore, err := m.startStore(ctx, run)
if err != nil {
return nil, fmt.Errorf("the integration level's store: %w", err)
}
defer stopStore()
nats, stopBus, err := m.startBus(ctx, filepath.Join(run, "nats"))
if err != nil {
return nil, fmt.Errorf("the integration level's bus: %w", err)
}
defer stopBus()
env := []string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}
// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
// below finds them built. Whether they compile is for the run to say; its failure is not read here.
for _, d := range dirs {
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", "integration", "-run", "^$", "-count=1", "./...")
}
setup := m.clock().Sub(began)
return m.runLevel(ctx, tree, levelOpts{
level: LevelIntegration, tags: []string{"integration"}, packages: packages, bounds: m.integrationBounds(),
env: env,
setup: setup, wall: true,
store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer,
})
}
modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go
+366
View File
@@ -0,0 +1,366 @@
package main
import (
"context"
"encoding/json"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// Issue 1486: the integration level, with the store and the bus faked at the process boundary: the commands the level
// runs are recorded, and a clock the commands advance stands in for the time they would take.
type integFake struct {
mu sync.Mutex
calls []cmdRecord
clock time.Time
postgres string // what `postgres --version` prints
captured string // the go test -json output of the run
// advance is how long a phase takes on the fake clock: compile, start (of the store), test.
advance map[string]time.Duration
busURL string
busDir string
busDown bool
runDirs []string // run directories seen when the store was started, to be gone at the end
panicOn string // a phase that panics
}
func newIntegFake(t *testing.T, captured string) *integFake {
t.Helper()
raw, err := os.ReadFile(filepath.Join("testdata", "captured", captured+".json"))
if err != nil {
t.Fatal(err)
}
return &integFake{clock: time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC), postgres: "postgres (PostgreSQL) 18.6",
captured: string(raw), advance: map[string]time.Duration{}, busURL: "nats://127.0.0.1:45222"}
}
func (f *integFake) now() time.Time { f.mu.Lock(); defer f.mu.Unlock(); return f.clock }
func (f *integFake) tick(phase string) {
f.mu.Lock()
f.clock = f.clock.Add(f.advance[phase])
f.mu.Unlock()
if f.panicOn == phase {
panic("the " + phase + " phase panicked")
}
}
func (f *integFake) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
line := name + " " + strings.Join(args, " ")
f.mu.Lock()
f.calls = append(f.calls, cmdRecord{env: env, line: line})
f.mu.Unlock()
switch {
case name == "postgres":
return forgekey.Ran{Stdout: f.postgres + "\n"}
case name == "git" && len(args) > 2 && args[2] == "ls-files":
return forgekey.Ran{Stdout: "go.mod\n"}
case name == "go" && strings.Contains(line, "-run ^$"):
f.tick("compile")
case name == "go":
f.tick("test")
status := 0
if strings.Contains(f.captured, `"Action":"fail"`) {
status = 1
}
return forgekey.Ran{Stdout: f.captured, Status: status}
case name == "pg_ctl" && strings.HasSuffix(line, " start"):
f.tick("start")
for i, a := range args {
if a == "-D" {
f.runDirs = append(f.runDirs, filepath.Dir(filepath.Dir(args[i+1])))
}
}
}
return forgekey.Ran{}
}
func (f *integFake) startBus(_ context.Context, dir string) (string, func(), error) {
f.mu.Lock()
f.busDir = dir
f.mu.Unlock()
return f.busURL, func() { f.mu.Lock(); f.busDown = true; f.mu.Unlock() }, nil
}
func (f *integFake) lines() string {
f.mu.Lock()
defer f.mu.Unlock()
out := []string{}
for _, c := range f.calls {
out = append(out, c.line)
}
return strings.Join(out, "\n")
}
func integModule(t *testing.T, f *integFake) (*Module, *keptStates) {
t.Helper()
states := newStates()
m := moduleWith(states, "go1.27.1")
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
m.Run, m.Now, m.StartBus = f.run, f.now, f.startBus
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
t.Fatal(err)
}
return m, states
}
func runLevelTool(t *testing.T, m *Module, states *keptStates, level string) (TestResult, delivery.Step) {
t.Helper()
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name != "build_go_test" {
continue
}
if _, err := tool.Run(map[string]any{"workspace": "ws1", "level": level}); err != nil {
t.Fatal(err)
}
}
st := states.waitEnd(t)
var res TestResult
if st.State == delivery.StepPassed {
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatal(err)
}
}
return res, st
}
// 1486/AC2: the store's start carries listen_addresses=” and a socket directory under the run; the tests are given
// its address and the bus's, and the run's directory is in the workspace's.
func TestTheStoreListensOnASocketUnderTheRunAndNowhereElse(t *testing.T) {
f := newIntegFake(t, "slow")
m, states := integModule(t, f)
res, st := runLevelTool(t, m, states, "integration")
if st.State != delivery.StepPassed || !res.Passed || res.Level != "integration" {
t.Fatalf("the step ended %s (%s): %+v", st.State, st.Reason, res)
}
var start, initdb string
for _, c := range f.calls {
if strings.HasPrefix(c.line, "pg_ctl ") && strings.HasSuffix(c.line, " start") {
start = c.line
}
if strings.HasPrefix(c.line, "initdb ") {
initdb = c.line
}
}
run := f.runDirs[0]
if !strings.HasPrefix(run, filepath.Join(m.Work, "ws1", "run-")) {
t.Errorf("the run directory is %s, not under the workspace's", run)
}
_, afterK, _ := strings.Cut(start, "-k ")
socket, _, _ := strings.Cut(afterK, " ")
if len(socket)+len(socketName) >= 100 {
t.Errorf("the socket %q is too long for postgres", socket)
}
for _, want := range []string{"listen_addresses=''", "-k " + socket, "fsync=off", "-w"} {
if !strings.Contains(start, want) {
t.Errorf("the store's start %q lacks %q", start, want)
}
}
if strings.Contains(start, "listen_addresses='*'") || strings.Contains(start, "-h ") || strings.Contains(start, "port=") {
t.Errorf("the store listens on TCP: %s", start)
}
if !strings.Contains(initdb, "-D "+filepath.Join(run, "pg", "data")) || !strings.Contains(initdb, "--auth=trust") {
t.Errorf("initdb = %q", initdb)
}
var test cmdRecord
for _, c := range f.calls {
if strings.HasPrefix(c.line, "go ") && strings.Contains(c.line, "-json") {
test = c
}
}
if !strings.Contains(test.line, "-tags integration") {
t.Errorf("the run does not ask for the tag: %s", test.line)
}
env := map[string]string{}
for _, w := range test.env {
k, v, _ := strings.Cut(w, "=")
env[k] = v
}
u, err := url.Parse(env["MESH_TEST_POSTGRES"])
if err != nil || u.Scheme != "postgres" || u.Query().Get("host") != socket || u.Host != "" {
t.Errorf("MESH_TEST_POSTGRES = %q", env["MESH_TEST_POSTGRES"])
}
if strings.Count(strings.TrimPrefix(env["MESH_TEST_POSTGRES"], "postgres://"), "/") != 1 {
t.Errorf("MESH_TEST_POSTGRES = %q has a slash beyond the one before the database: the controller's tests cut it at the last", env["MESH_TEST_POSTGRES"])
}
if env["MESH_TEST_NATS"] != "nats://127.0.0.1:45222" || f.busDir != filepath.Join(run, "nats") {
t.Errorf("MESH_TEST_NATS = %q, bus dir %q", env["MESH_TEST_NATS"], f.busDir)
}
if res.Store != "postgres 18" || res.Bus != "nats-server "+m.Recipe.NatsServer {
t.Errorf("the result names %q and %q", res.Store, res.Bus)
}
}
// 1486/AC3: a postgres of major 17 refuses the integration level, naming 17 and 18; the unit level still runs.
func TestAnotherPostgresMajorRefusesTheIntegrationLevelOnly(t *testing.T) {
f := newIntegFake(t, "slow")
f.postgres = "postgres (PostgreSQL) 17.2"
m, states := integModule(t, f)
_, st := runLevelTool(t, m, states, "integration")
if st.State != delivery.StepRefused || !strings.Contains(st.Reason, "17") || !strings.Contains(st.Reason, "18") {
t.Fatalf("the integration step ended %s: %q", st.State, st.Reason)
}
if strings.Contains(f.lines(), "initdb") || strings.Contains(f.lines(), "-tags") {
t.Errorf("the refused level ran something:\n%s", f.lines())
}
res, st := runLevelTool(t, m, states, "unit")
if st.State != delivery.StepPassed || !res.Passed || res.Level != "unit" {
t.Errorf("the unit level: %s %+v", st.State, res)
}
}
// 1486/AC4: a setup over thirty seconds fails the level naming the setup bound; a run over sixty names the running
// bound and the slowest tests. The clock the commands move stands in for the time they take.
func TestSetupAndRunningTimeAreJudgedApart(t *testing.T) {
f := newIntegFake(t, "slow")
f.advance["compile"], f.advance["start"] = 20*time.Second, 15*time.Second
m, states := integModule(t, f)
res, _ := runLevelTool(t, m, states, "integration")
if !res.Passed || !res.OverBound || res.OverWhich != "setup" || res.Setup != 35 || !strings.Contains(res.Why, "setup took 35.00 s, over its setup bound of 30 s") || strings.Contains(res.Why, "slowest tests") {
t.Errorf("a setup of 35 s: %+v", res)
}
g := newIntegFake(t, "slow")
g.advance["test"] = 61 * time.Second
m, states = integModule(t, g)
res, _ = runLevelTool(t, m, states, "integration")
if !res.Passed || res.OverWhich != "run" || res.Elapsed != 61 || !strings.Contains(res.Why, "took 61.00 s, over its bound of 60 s; the slowest tests: fixture/slow TestTakesAWhile") || strings.Contains(res.Why, "setup") {
t.Errorf("a run of 61 s: %+v", res)
}
h := newIntegFake(t, "slow")
h.advance["compile"], h.advance["test"] = 29*time.Second, 59*time.Second
m, states = integModule(t, h)
if res, _ = runLevelTool(t, m, states, "integration"); !res.Passed || res.OverBound || res.Setup != 29 || res.Elapsed != 59 {
t.Errorf("29 s of setup and 59 s of running are within both bounds: %+v", res)
}
}
// 1486/AC5: a run that fails, and a run that panics: the store is stopped, the bus is stopped, the run's directory is
// gone.
func TestAFailedOrPanickedRunStopsTheServersAndRemovesTheRunDirectory(t *testing.T) {
f := newIntegFake(t, "failing")
m, states := integModule(t, f)
res, _ := runLevelTool(t, m, states, "integration")
if res.Passed {
t.Fatalf("the failing fixture passed: %+v", res)
}
checkCleaned := func(f *integFake, m *Module) {
t.Helper()
lines := f.lines()
if !strings.Contains(lines, "pg_ctl -D "+filepath.Join(f.runDirs[0], "pg", "data")+" -m immediate -w stop") {
t.Errorf("the store was not stopped:\n%s", lines)
}
if strings.LastIndex(lines, " stop") < strings.LastIndex(lines, "-json") {
t.Errorf("the store was stopped before the tests ended:\n%s", lines)
}
if !f.busDown {
t.Errorf("the bus was not stopped")
}
if _, err := os.Stat(f.runDirs[0]); !os.IsNotExist(err) {
t.Errorf("the run directory %s is still there (%v)", f.runDirs[0], err)
}
}
checkCleaned(f, m)
g := newIntegFake(t, "failing")
g.panicOn = "test"
m, _ = integModule(t, g)
func() {
defer func() { recover() }()
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil)
}()
checkCleaned(g, m)
}
// A store that does not start ends the step failed, with the server stopped and the directory gone.
func TestAStoreThatDoesNotStartEndsTheStepFailedAndLeavesNothing(t *testing.T) {
f := newIntegFake(t, "slow")
m, states := integModule(t, f)
inner := m.Run
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
r := inner(ctx, env, name, args...)
if name == "pg_ctl" && strings.HasSuffix(strings.Join(args, " "), " start") {
return forgekey.Ran{Status: 1, Stderr: "pg_ctl: could not start server\n"}
}
return r
}
_, st := runLevelTool(t, m, states, "integration")
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "pg_ctl: could not start server") {
t.Errorf("the step ended %s: %q", st.State, st.Reason)
}
if !strings.Contains(f.lines(), " stop") {
t.Errorf("the half-started store was not stopped:\n%s", f.lines())
}
if f.busDir != "" {
t.Errorf("the bus was started after the store failed")
}
if entries, _ := os.ReadDir(filepath.Join(m.Work, "ws1")); len(entries) != 1 || entries[0].Name() != "tree" {
t.Errorf("the workspace holds %v, want only its tree", entries)
}
}
func TestThePostgresMajorIsReadFromTheVersionLine(t *testing.T) {
for line, want := range map[string]int{"postgres (PostgreSQL) 18.6": 18, "postgres (PostgreSQL) 9.6.24": 9, "postgres (PostgreSQL) 17.2 (Arch)": 17} {
f := &integFake{postgres: line, advance: map[string]time.Duration{}}
m := &Module{Run: f.run}
if got, err := m.postgresMajor(context.Background()); err != nil || got != want {
t.Errorf("%q = %d, %v; want %d", line, got, err, want)
}
}
if _, err := (&Module{Run: (&integFake{postgres: "no", advance: map[string]time.Duration{}}).run}).postgresMajor(context.Background()); err == nil {
t.Errorf("accepted a line that is not postgres's")
}
}
// Issue 1486 review: postgres refuses a socket path of 108 bytes and over. A run directory deep in the work directory
// gets a short directory of its own for the socket, removed with the store; a short one keeps it under the run.
func TestTheSocketPathStaysUnderTheLimitForALongRunDirectory(t *testing.T) {
long := filepath.Join(t.TempDir(), strings.Repeat("deep", 20), "ws1", "run-mvh5uc96142806")
if err := os.MkdirAll(long, 0o700); err != nil {
t.Fatal(err)
}
dir, short, err := socketDir(long)
if err != nil || !short || len(dir)+len(socketName) >= 100 {
t.Fatalf("socketDir(long) = %q, %v, %v", dir, short, err)
}
if _, err := os.Stat(dir); err != nil {
t.Errorf("the short directory was not made: %v", err)
}
os.RemoveAll(dir)
near := t.TempDir()
if dir, short, err := socketDir(near); err != nil || (len(near) < 50 && (short || !strings.HasPrefix(dir, near))) {
t.Errorf("socketDir(short) = %q, %v, %v", dir, short, err)
}
// Through the store: the short directory is removed when the store is stopped.
f := newIntegFake(t, "slow")
m, _ := integModule(t, f)
_, stop, err := m.startStore(context.Background(), long)
if err != nil {
t.Fatal(err)
}
var sock string
for _, c := range f.calls {
if _, a, ok := strings.Cut(c.line, "-k "); ok && strings.HasPrefix(c.line, "pg_ctl") {
sock, _, _ = strings.Cut(a, " ")
}
}
if sock == "" || len(sock)+len(socketName) >= 100 {
t.Fatalf("the store was started with the socket %q", sock)
}
stop()
if _, err := os.Stat(sock); !os.IsNotExist(err) {
t.Errorf("the short socket directory %s outlived the store (%v)", sock, err)
}
}
modules/mesh-build-go/cmd/mesh-build-go/keys.go
+42
View File
@@ -0,0 +1,42 @@
package main
// The module's own forge key (novox/hq ADR 0466 §2, issue 1484): mesh-sdk's key keeper, kind builder, for the
// module's account in its home. It makes ~/.ssh/mesh_ed25519 when it is missing and writes one entry to this
// module's `keys` state at <node>.<account>, which gitea's provider reads to register the key on the forge user
// that reads and pushes nothing (ADR 0466 §3).
import (
"context"
"fmt"
"os"
"os/user"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// keeperFor is this module's keeper: go/forgekey's, of kind builder, for account in home on node, whose entry lies
// at <node>.<account> of the keys state.
func keeperFor(home, node, account string, run forgekey.Runner, state forgekey.KeyPutter) *forgekey.KeyKeeper {
return &forgekey.KeyKeeper{Home: home, Node: node, Account: account, Kind: forgekey.KindBuilder, Run: run, State: state, Now: time.Now}
}
// startKeeper runs the keeper when the runtime launched this process for a node: outside a launch — a build, a
// check — there is no state to put the entry in.
func startKeeper(ctx context.Context) {
node := strings.TrimSpace(os.Getenv("MESH_NODE"))
u, err := user.Current()
if node == "" || err != nil || u.HomeDir == "" {
return
}
k := keeperFor(u.HomeDir, node, u.Username, forgekey.ExecRunner, keysState{})
k.Keep(ctx, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-go] "+f+"\n", a...) })
}
type keysState struct{}
func (keysState) Put(key string, value any) (uint64, error) {
return stdio.State("keys").Put(key, value)
}
modules/mesh-build-go/cmd/mesh-build-go/levels.go
+262
View File
@@ -0,0 +1,262 @@
package main
// The judging of a test level (novox/hq ADR 0445 §2 §3; issues 1290 and 1486): `go test -json` read into a result
// that says, per package, what ran and for how long, which tests failed and why, and whether the level kept its
// bound. A level that failed is still a finished step: the verdict is in the result, and a step that ends failed is a
// step that could not run.
import (
"bufio"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
)
// The two levels (ADR 0445 §2).
const (
LevelUnit = "unit"
LevelIntegration = "integration"
)
// UnitBound is the unit level's one bound: its running time, five seconds for the level, no setup (ADR 0445 §3). The
// level's time is that of its slowest package, as packages run side by side, the compile not counted.
const UnitBound = 5 * time.Second
// Bounds are the bounds a level is judged by: Setup is the time of everything before the tests run (compiling,
// starting the store and the bus; the unit level has none) and Run the tests' own time.
type Bounds struct {
Setup, Run time.Duration
}
// PackageResult is one package's part of a level.
type PackageResult struct {
Dir string `json:"dir"`
Package string `json:"package"`
Passed bool `json:"passed"`
Elapsed float64 `json:"elapsed"`
Tests int `json:"tests"`
// NoTests is a package with no test files (at this level: a file with the other level's build tag is none).
NoTests bool `json:"no_tests,omitempty"`
}
// TestTime is one test and the time it took.
type TestTime struct {
Package string `json:"package"`
Test string `json:"test"`
Elapsed float64 `json:"elapsed"`
}
// Failure is a test that failed, or a package that failed with no test to name (a build that failed, a TestMain that
// panicked); Output is the first lines it wrote.
type Failure struct {
Package string `json:"package"`
Test string `json:"test,omitempty"`
Output string `json:"output,omitempty"`
}
// TestResult is what a test step passes with.
type TestResult struct {
Level string `json:"level"`
Passed bool `json:"passed"`
// Why says, in words, why the level failed, or that it has no tests.
Why string `json:"why,omitempty"`
NoTests bool `json:"no_tests,omitempty"`
Tests int `json:"tests"`
Skipped int `json:"skipped"`
Packages []PackageResult `json:"packages"`
// Elapsed is the level's running time in seconds: the slowest package's.
Elapsed float64 `json:"elapsed"`
// Setup is the level's setup time in seconds, when it has a setup.
Setup float64 `json:"setup,omitempty"`
// Bound and SetupBound are the bounds the level was judged by, in seconds.
Bound float64 `json:"bound"`
SetupBound float64 `json:"setup_bound,omitempty"`
Slowest []TestTime `json:"slowest,omitempty"`
Failed []Failure `json:"failed,omitempty"`
// Store and Bus are what the integration level ran against: the postgres major version and the nats-server
// version (ADR 0470 §4).
// OverBound says a bound was kept no longer, and OverWhich which: setup, run or both. The level's verdict, Passed, is
// the tests' own: the checker judges the bounds (ADR 0453 §9), with the slowest tests this result names.
OverBound bool `json:"over_bound"`
OverWhich string `json:"over_which,omitempty"`
Store string `json:"store,omitempty"`
Bus string `json:"bus,omitempty"`
}
// event is one line of `go test -json` (test2json, and the build events go test adds).
type event struct {
Action string
Package string
Test string
Output string
ImportPath string
FailedBuild string
Elapsed float64
}
// outputLines is how many lines of output a failure keeps.
const outputLines = 15
// reader collects the events of one go.mod directory's `go test -json` into a result.
type reader struct {
res *TestResult
dir string
pkgs map[string]*PackageResult
order []string
builds map[string][]string // build output by import path
pkgOut map[string][]string
testOut map[string][]string // by package + "\x00" + test
times []TestTime
failedIn map[string]bool // packages with a failed test
}
func newReader(res *TestResult, dir string) *reader {
return &reader{res: res, dir: dir, pkgs: map[string]*PackageResult{}, builds: map[string][]string{},
pkgOut: map[string][]string{}, testOut: map[string][]string{}, failedIn: map[string]bool{}}
}
// read takes go test's output. Lines that are not events are kept for a failure that has nothing else to say.
func (r *reader) read(out string) (noise []string) {
sc := bufio.NewScanner(strings.NewReader(out))
sc.Buffer(make([]byte, 1<<20), 1<<26)
for sc.Scan() {
var ev event
if err := json.Unmarshal(sc.Bytes(), &ev); err != nil {
if l := strings.TrimSpace(sc.Text()); l != "" {
noise = append(noise, l)
}
continue
}
r.event(ev)
}
return noise
}
func (r *reader) pkg(name string) *PackageResult {
p := r.pkgs[name]
if p == nil {
p = &PackageResult{Dir: r.dir, Package: name}
r.pkgs[name] = p
r.order = append(r.order, name)
}
return p
}
func keep(lines []string, out string) []string {
if t := strings.TrimRight(out, "\r\n"); t != "" && !strings.HasPrefix(t, "=== ") {
lines = append(lines, t)
}
return lines
}
func (r *reader) event(ev event) {
switch {
case ev.Package == "" && ev.ImportPath != "":
if ev.Action == "build-output" {
r.builds[ev.ImportPath] = keep(r.builds[ev.ImportPath], ev.Output)
}
case ev.Package == "":
case ev.Test == "":
p := r.pkg(ev.Package)
switch ev.Action {
case "output":
r.pkgOut[ev.Package] = keep(r.pkgOut[ev.Package], ev.Output)
case "pass":
p.Passed, p.Elapsed = true, ev.Elapsed
case "fail":
p.Passed, p.Elapsed = false, ev.Elapsed
if ev.FailedBuild != "" {
r.pkgOut[ev.Package] = append(r.builds[ev.FailedBuild], r.pkgOut[ev.Package]...)
}
case "skip":
p.Passed, p.NoTests = true, true
}
default:
key := ev.Package + "\x00" + ev.Test
switch ev.Action {
case "output":
r.testOut[key] = keep(r.testOut[key], ev.Output)
case "pass", "fail", "skip":
p := r.pkg(ev.Package)
p.Tests++
r.res.Tests++
if ev.Action == "skip" {
r.res.Skipped++
}
r.times = append(r.times, TestTime{Package: ev.Package, Test: ev.Test, Elapsed: ev.Elapsed})
if ev.Action == "fail" {
r.failedIn[ev.Package] = true
r.res.Failed = append(r.res.Failed, Failure{Package: ev.Package, Test: ev.Test, Output: clip(r.testOut[key])})
}
}
}
}
func clip(lines []string) string {
if len(lines) > outputLines {
lines = append(lines[:outputLines:outputLines], fmt.Sprintf("... (%d more lines)", len(lines)-outputLines))
}
return strings.Join(lines, "\n")
}
// finish moves what was read into the result: the packages in order, a failure for each failed package that has no
// failed test to name it, and the tests' times.
func (r *reader) finish() {
for _, name := range r.order {
p := r.pkgs[name]
if !p.Passed && !r.failedIn[name] {
r.res.Failed = append(r.res.Failed, Failure{Package: name, Output: clip(r.pkgOut[name])})
}
if !p.NoTests && p.Elapsed > r.res.Elapsed {
r.res.Elapsed = p.Elapsed
}
r.res.Packages = append(r.res.Packages, *p)
}
r.res.Slowest = append(r.res.Slowest, r.times...)
}
// judge sets Passed and Why from what was read and the bounds. A level with no failure, over no bound, passes; with no
// test at all it says the repository has none at this level, which is neither a failure nor a warning (ADR 0445 §1).
func (res *TestResult) judge(b Bounds) {
sort.SliceStable(res.Slowest, func(i, j int) bool { return res.Slowest[i].Elapsed > res.Slowest[j].Elapsed })
if len(res.Slowest) > 5 {
res.Slowest = res.Slowest[:5]
}
res.Bound, res.SetupBound = b.Run.Seconds(), b.Setup.Seconds()
var why, over []string
if n := len(res.Failed); n > 0 {
names := []string{}
for _, f := range res.Failed {
names = append(names, strings.TrimSpace(f.Package+" "+f.Test))
}
why = append(why, fmt.Sprintf("%d failed: %s", n, strings.Join(names, ", ")))
}
res.Passed = len(res.Failed) == 0
if b.Setup > 0 && res.Setup > b.Setup.Seconds() {
over = append(over, "setup")
why = append(why, fmt.Sprintf("the %s level's setup took %.2f s, over its setup bound of %s", res.Level, res.Setup, seconds(b.Setup)))
}
if b.Run > 0 && res.Elapsed > b.Run.Seconds() {
over = append(over, "run")
slow := []string{}
for _, s := range res.Slowest {
slow = append(slow, fmt.Sprintf("%s %s (%.2f s)", s.Package, s.Test, s.Elapsed))
}
why = append(why, fmt.Sprintf("the %s level took %.2f s, over its bound of %s; the slowest tests: %s", res.Level, res.Elapsed, seconds(b.Run), strings.Join(slow, ", ")))
}
res.OverBound, res.OverWhich = len(over) > 0, strings.Join(over, ",")
res.NoTests = res.Tests == 0 && len(res.Failed) == 0
switch {
case len(why) > 0:
res.Why = strings.Join(why, "; ")
case res.NoTests:
res.Why = "the repository has no tests at the " + res.Level + " level"
}
}
// seconds is a duration as seconds, without trailing zeros: "5 s", "0.01 s".
func seconds(d time.Duration) string {
return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.3f", d.Seconds()), "0"), ".") + " s"
}
modules/mesh-build-go/cmd/mesh-build-go/main.go
+58
View File
@@ -0,0 +1,58 @@
// mesh-build-go: the Go build module (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470; design 47 R4c-3, issue
// 1483). A Go tools bundle the node's runtime launches as the module's own account (runs-as). It names the one Go
// toolchain version it builds with, writes the version it finds to <node>.go of its `toolchains` state at start and
// before every step, and refuses every step on a node that has another: its health is then unhealthy, so the
// controller raises the node's condition. What it says, it says on stderr; stdout is the MCP channel.
package main
import (
"context"
"fmt"
"os"
"os/user"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The directories the module's manifest places outside its home (ADR 0466 §1): the workspaces, and the Go module
// and build caches.
const (
WorkDir = "/var/lib/mesh-build-go-work"
CacheDir = "/var/lib/mesh-build-go-cache"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-go] "+format+"\n", a...) }
func main() {
if len(os.Args) > 1 && os.Args[1] == busArg {
os.Exit(runBus(os.Args[2:]))
}
ctx := context.Background()
m := &Module{
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
Recipe: Current(),
Probe: ProbeGo,
Toolchains: stdio.State(delivery.ToolchainsState).Put,
Emit: func(t string, b any) error { return stdio.Emit(t, b) },
Steps: stdio.State(delivery.StepsState).Put,
Ctx: ctx,
Say: logf,
Work: WorkDir,
Cache: CacheDir,
}
if u, err := user.Current(); err == nil {
m.Home = u.HomeDir
}
m.Run = StepRunner(m.Home)
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
if m.Node != "" {
go m.Start(ctx)
}
go startKeeper(ctx)
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
logf("%v", err)
os.Exit(1)
}
}
modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go
+150
View File
@@ -0,0 +1,150 @@
package main
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The catalogue's root and this module's directory.
const (
catalogueRoot = "../../../.."
moduleDir = "../.."
)
func readManifest(t *testing.T) catalogue.Manifest {
t.Helper()
m, err := catalogue.ReadTreeManifest(catalogueRoot, moduleDir)
if err != nil {
t.Fatalf("mesh-sdk's validator refuses module.json: %v", err)
}
return m
}
func resource(t *testing.T, m catalogue.Manifest, id string) map[string]any {
t.Helper()
for _, r := range m.Resources {
if r["id"] == id {
return r
}
}
t.Fatalf("module.json declares no resource %q", id)
return nil
}
// 1483/AC1: mesh-sdk's validator accepts the module (alone on a shelf too), and it runs as mesh-build-go with its
// home at mode 0700, requires user-namespaces and declares the four packages.
func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *testing.T) {
m := readManifest(t)
if problems := catalogue.CatalogueProblems(catalogue.Shelf{m.Module: m}); len(problems) > 0 {
t.Errorf("the catalogue's rules refuse it: %v", problems)
}
if m.Module != "mesh-build-go" || m.RunsAs != "mesh-build-go" {
t.Errorf("module %q runs as %q, want mesh-build-go as itself", m.Module, m.RunsAs)
}
account := resource(t, m, "account")
if account["type"] != "user" || account["name"] != "mesh-build-go" || account["shell"] != "/usr/bin/nologin" || account["home"] != "/var/lib/mesh-build-go" {
t.Errorf("account = %v", account)
}
home := resource(t, m, "state")
if home["type"] != "directory" || home["mode"] != "0700" || home["owner"] != "mesh-build-go" || home["place"] != "." {
t.Errorf("home = %v, want its directory at mode 0700 owned by the account", home)
}
if !slices.Contains(m.Capabilities, "user-namespaces") {
t.Errorf("capabilities = %v, want user-namespaces", m.Capabilities)
}
packages := []string{}
for _, r := range m.Resources {
if r["type"] == "package" {
packages = append(packages, r["package"].(string))
}
}
slices.Sort(packages)
if want := []string{"git", "go", "openssh", "postgresql"}; !slices.Equal(packages, want) {
t.Errorf("packages = %v, want %v", packages, want)
}
}
// The work and cache directories are outside the home (ADR 0466 §1), owned by the account.
func TestWorkAndCacheDirectoriesLieOutsideTheHome(t *testing.T) {
m := readManifest(t)
for _, id := range []string{"work", "cache"} {
r := resource(t, m, id)
path, _ := r["path"].(string)
if path == "" || path == "/var/lib/mesh-build-go" || len(path) > len("/var/lib/mesh-build-go") && path[:len("/var/lib/mesh-build-go/")] == "/var/lib/mesh-build-go/" {
t.Errorf("%s is at %q, inside or at the home", id, path)
}
if r["owner"] != "mesh-build-go" {
t.Errorf("%s is owned by %v", id, r["owner"])
}
}
}
// The states are the three the module keeps, no seat is claimed (ADR 0469 §6), and the health is the module's own
// tool, which the manifest also lists among its tools.
func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) {
m := readManifest(t)
states := []string{}
for _, s := range m.State {
states = append(states, s.Name)
}
slices.Sort(states)
if want := []string{"keys", "steps", "toolchains"}; !slices.Equal(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if len(m.Claims) != 0 {
t.Errorf("claims = %v, want no seat", m.Claims)
}
if !slices.Contains(m.Tools, HealthTool) {
t.Errorf("tools = %v lack %s", m.Tools, HealthTool)
}
raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json"))
if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) {
t.Errorf("runtime-health does not name %s", HealthTool)
}
}
// Every tool the manifest names is served, and every tool served is named.
func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
m := readManifest(t)
served := []string{}
for _, tool := range (&Module{}).Tools((&Module{}).StepTools()) {
served = append(served, tool.Name)
}
slices.Sort(served)
named := slices.Clone(m.Tools)
slices.Sort(named)
if !slices.Equal(served, named) {
t.Errorf("served %v, manifest names %v", served, named)
}
}
// 1482/AC4: gitea's manifest reads the keys state of both build modules, and the catalogue's validator accepts it with the
// build modules beside it: each owner present declares the state read (a read of a state its owner does not keep is
// refused).
func TestGiteaReadsTheBuildModulesKeysAndTheCatalogueAcceptsThemTogether(t *testing.T) {
var all []catalogue.Manifest
var gitea catalogue.Manifest
for _, name := range []string{"gitea", "mesh-build-go", "mesh-build-node"} {
m, err := catalogue.ReadTreeManifest(catalogueRoot, filepath.Join(catalogueRoot, "modules", name))
if err != nil {
t.Fatalf("mesh-sdk's validator refuses modules/%s/module.json: %v", name, err)
}
all = append(all, m)
if name == "gitea" {
gitea = m
}
}
for _, want := range []string{"mesh-build-go.keys", "mesh-build-node.keys", "ssh-client.keys"} {
if !slices.Contains(gitea.Reads, want) {
t.Errorf("gitea reads %v, lacks %s", gitea.Reads, want)
}
}
if problems := catalogue.StateReadsNothingDeclares(all); len(problems) > 0 {
t.Errorf("a read names a state its owner does not keep: %v", problems)
}
}
modules/mesh-build-go/cmd/mesh-build-go/module.go
+146
View File
@@ -0,0 +1,146 @@
package main
import (
"context"
"fmt"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
"git.novox.be/novox/mesh-sdk/go/health"
)
// CheckRound is how often the module reads the installed toolchain again, beside the check before every step and
// the one every health look makes.
const CheckRound = 10 * time.Minute
// Module is the Go build module's one running state: the node it runs on, its recipe, the toolchain guard and the
// states it writes (toolchains, steps).
type Module struct {
Node string
Recipe Recipe
// Probe reads the installed Go version (ProbeGo).
Probe func(ctx context.Context) (string, error)
// Toolchains and Steps are the Put of the states of those names.
Toolchains, Steps buildstep.Put
// Emit tells the bus an event (stdio.Emit); nil emits nothing. The guard emits toolchain.reported after each write of
// the toolchains key, so the indexer, which holds no timer, learns that the bucket exists (issue 1444).
Emit func(eventType string, body any) error
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
Ctx context.Context
Say func(format string, a ...any)
// Run runs the commands of a step (StepRunner); Work holds the workspaces and Cache the module and build caches,
// both outside the account's home (ADR 0466 §1); Home is the account's home.
Run forgekey.Runner
Home, Work, Cache string
// UnitBound is the unit level's bound; zero is UnitBound. A test lowers it to see a level over its bound.
UnitBound time.Duration
// IntegrationBounds are the integration level's bounds; zero values are IntegrationSetup and IntegrationRun.
IntegrationBounds Bounds
// Now is the clock the levels' times are read from; time.Now when nil.
Now func() time.Time
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
// program, StartBus in bus.go, when nil).
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
}
func (m *Module) guard() buildstep.Guard {
return buildstep.Guard{Language: "go", Want: m.Recipe.Go, Recipe: m.Recipe.Version(), Probe: m.Probe, Node: m.Node, Put: m.reportingPut()}
}
// Check reads the installed toolchain, writes <node>.go of the toolchains state and answers nil, or the refusal
// (buildstep.Guard.Check). It runs at start, before every step, at every health look and every CheckRound.
func (m *Module) Check(ctx context.Context) error {
return m.guard().Check(ctx)
}
// Start checks once, as the module starts, and then every CheckRound until ctx ends.
func (m *Module) Start(ctx context.Context) {
for {
if err := m.Check(ctx); err != nil && m.Say != nil {
m.Say("the toolchain: %v", err)
}
m.CleanWorkspaces(time.Now())
select {
case <-ctx.Done():
return
case <-time.After(CheckRound):
}
}
}
// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy while the toolchain installed is not
// the recipe's, saying both versions, so the controller raises the node's condition. It looks again each time it is
// asked.
func (m *Module) Health(ctx context.Context) health.Answer {
err := m.Check(ctx)
switch {
case err == nil:
return health.Healthy(fmt.Sprintf("%s installed, as the recipe %s builds with", m.Recipe.Go, m.Recipe.Version()))
default:
return health.Unhealthy(err.Error())
}
}
// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on
// its key in the steps state. The guard runs inside the step, before run, so a node that refuses ends every step
// refused with both versions named.
func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
return func(args map[string]any) (any, error) {
id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) {
if err := m.Check(ctx); err != nil {
return nil, err
}
return run(ctx, args)
})
if err != nil {
return nil, err
}
key, _ := delivery.StepKey(m.Node, id)
return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil
}
}
// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error.
func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
return func(args map[string]any) (any, error) {
if err := m.Check(m.Ctx); err != nil {
return nil, err
}
return run(m.Ctx, args)
}
}
// Status is what the module reports of itself: the node, the recipe, the toolchain found and why it refuses, if it
// does.
func (m *Module) Status(ctx context.Context) map[string]any {
err := m.Check(ctx)
installed, _ := m.Probe(ctx)
out := map[string]any{
"node": m.Node, "recipe": m.Recipe.Version(), "go": m.Recipe.Go, "installed": installed,
"postgres-major": m.Recipe.PostgresMajor, "nats-server": m.Recipe.NatsServer, "refusing": err != nil,
}
if err != nil {
out["why"] = err.Error()
}
return out
}
// ToolchainReported is the event the guard emits after it writes the toolchains key.
const ToolchainReported = "toolchain.reported"
// reportingPut is the toolchains Put that then emits toolchain.reported with the node and the language. A write that
// failed emits nothing; an emit that failed is said and does not refuse the step.
func (m *Module) reportingPut() buildstep.Put {
return func(key string, value any) (uint64, error) {
rev, err := m.Toolchains(key, value)
if err != nil || m.Emit == nil {
return rev, err
}
if eerr := m.Emit(ToolchainReported, map[string]string{"node": m.Node, "language": "go"}); eerr != nil && m.Say != nil {
m.Say("the event %s could not be emitted: %v", ToolchainReported, eerr)
}
return rev, nil
}
}
modules/mesh-build-go/cmd/mesh-build-go/module_test.go
+201
View File
@@ -0,0 +1,201 @@
package main
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// keptStates stands in for the toolchains and the steps states.
type keptStates struct {
mu sync.Mutex
toolchains map[string]delivery.Toolchain
steps map[string][]delivery.Step
ended chan string
}
func newStates() *keptStates {
return &keptStates{toolchains: map[string]delivery.Toolchain{}, steps: map[string][]delivery.Step{}, ended: make(chan string, 16)}
}
func (s *keptStates) putToolchain(key string, value any) (uint64, error) {
tc, err := delivery.DecodeToolchain(value.(json.RawMessage))
if err != nil {
return 0, err
}
s.mu.Lock()
defer s.mu.Unlock()
s.toolchains[key] = tc
return 1, nil
}
func (s *keptStates) putStep(key string, value any) (uint64, error) {
st, err := delivery.DecodeStep(value.(json.RawMessage))
if err != nil {
return 0, err
}
s.mu.Lock()
s.steps[key] = append(s.steps[key], st)
s.mu.Unlock()
if st.State != delivery.StepRunning {
s.ended <- key
}
return 1, nil
}
func (s *keptStates) toolchain(key string) (delivery.Toolchain, bool) {
s.mu.Lock()
defer s.mu.Unlock()
tc, ok := s.toolchains[key]
return tc, ok
}
func (s *keptStates) waitEnd(t *testing.T) delivery.Step {
t.Helper()
select {
case key := <-s.ended:
s.mu.Lock()
defer s.mu.Unlock()
v := s.steps[key]
return v[len(v)-1]
case <-time.After(3 * time.Second):
t.Fatal("the step never ended on its key")
return delivery.Step{}
}
}
func moduleWith(states *keptStates, installed string) *Module {
return &Module{Node: "ace", Recipe: Recipe{Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"},
Probe: func(context.Context) (string, error) { return installed, nil },
Toolchains: states.putToolchain, Steps: states.putStep, Ctx: context.Background()}
}
// 1483/AC2: a probe answering go1.27.1: the bundle's start writes ace.go with that version and the recipe's, and
// the health tool answers healthy.
func TestStartWritesTheToolchainAndHealthIsHealthy(t *testing.T) {
states := newStates()
m := moduleWith(states, "go1.27.1")
ctx, stop := context.WithCancel(context.Background())
defer stop()
go m.Start(ctx)
var tc delivery.Toolchain
for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
if got, ok := states.toolchain("ace.go"); ok {
tc = got
break
}
}
if tc.Version != "go1.27.1" || tc.Recipe != m.Recipe.Version() || tc.Language != "go" {
t.Fatalf("ace.go = %+v, want go1.27.1 and recipe %s", tc, m.Recipe.Version())
}
if a := m.Health(ctx); a.State != health.StateHealthy || !a.Healthy {
t.Errorf("health = %+v, want healthy", a)
}
}
// 1483/AC3: a probe answering go1.27.2: every step tool's step ends refused naming both versions, ace.go holds
// go1.27.2, and the health tool answers unhealthy naming both.
func TestAnotherGoRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
states := newStates()
m := moduleWith(states, "go1.27.2")
ran := false
run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
steps := append([]StepTool{
{Name: "long_probe", Long: true, Run: run},
{Name: "short_probe", Run: run},
}, m.StepTools()...)
for _, tool := range m.Tools(steps) {
if tool.Name == StatusTool || tool.Name == HealthTool {
continue
}
var reason string
out, err := tool.Run(map[string]any{})
if isLong(steps, tool.Name) {
if err != nil {
t.Fatalf("%s: %v", tool.Name, err)
}
if out.(map[string]any)["step"] == "" {
t.Fatalf("%s answered no step id: %v", tool.Name, out)
}
st := states.waitEnd(t)
if st.State != delivery.StepRefused {
t.Errorf("%s: the step ended %s, want refused", tool.Name, st.State)
}
reason = st.Reason
} else {
if err == nil {
t.Fatalf("%s: answered %v, want a refusal", tool.Name, out)
}
reason = err.Error()
}
if !strings.Contains(reason, "go1.27.2") || !strings.Contains(reason, "go1.27.1") {
t.Errorf("%s: the refusal %q does not name both versions", tool.Name, reason)
}
}
if ran {
t.Errorf("a step ran on a node that refuses")
}
if tc, _ := states.toolchain("ace.go"); tc.Version != "go1.27.2" {
t.Errorf("ace.go holds %q, want the installed go1.27.2", tc.Version)
}
a := m.Health(context.Background())
if a.State != health.StateUnhealthy || !strings.Contains(a.Why, "go1.27.2") || !strings.Contains(a.Why, "go1.27.1") {
t.Errorf("health = %+v, want unhealthy naming both versions", a)
}
}
func isLong(steps []StepTool, name string) bool {
for _, s := range steps {
if s.Name == name {
return s.Long
}
}
return false
}
// 1483/AC4: recipes differing only in the postgres major or the nats-server version have different recipe versions;
// equal recipes have the same one.
func TestRecipeVersionMovesWithEachOfItsValues(t *testing.T) {
base := Recipe{Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"}
if base.Version() != (Recipe{Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"}).Version() {
t.Errorf("one recipe has two versions")
}
seen := map[string]string{base.Version(): "base"}
for name, r := range map[string]Recipe{
"postgres major": {Steps: "1", Go: "go1.27.1", PostgresMajor: 19, NatsServer: "2.11.17"},
"nats-server": {Steps: "1", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.18"},
"go": {Steps: "1", Go: "go1.27.2", PostgresMajor: 18, NatsServer: "2.11.17"},
"steps": {Steps: "2", Go: "go1.27.1", PostgresMajor: 18, NatsServer: "2.11.17"},
} {
v := r.Version()
if was, dup := seen[v]; dup {
t.Errorf("the recipe differing in %s has the version of the %s recipe: %s", name, was, v)
}
seen[v] = name
}
}
func TestTheRecipeNamesGo1271AndPostgres18AndTheLibrarysNatsVersion(t *testing.T) {
r := Current()
if r.Go != "go1.27.1" || r.PostgresMajor != 18 {
t.Errorf("recipe = %+v", r)
}
if r.NatsServer == "" || strings.HasPrefix(r.NatsServer, "v") {
t.Errorf("nats-server version = %q", r.NatsServer)
}
}
func TestParseGoVersion(t *testing.T) {
if v, err := parseGoVersion("go version go1.27.1 linux/amd64\n"); err != nil || v != "go1.27.1" {
t.Errorf("= %q, %v", v, err)
}
if _, err := parseGoVersion("not go"); err == nil {
t.Errorf("accepted a text that is not `go version`'s")
}
}
modules/mesh-build-go/cmd/mesh-build-go/probe.go
+36
View File
@@ -0,0 +1,36 @@
package main
import (
"context"
"fmt"
"os"
"os/exec"
"strings"
"time"
)
// ProbeTimeout bounds `go version`.
const ProbeTimeout = 15 * time.Second
// ProbeGo reads the installed Go version: the third word `go version` prints ("go version go1.27.1 linux/amd64"),
// as the recipe's RecipeGo is written. GOTOOLCHAIN=local keeps an older Go from fetching another toolchain: what is
// installed is what is read.
func ProbeGo(ctx context.Context) (string, error) {
ctx, cancel := context.WithTimeout(ctx, ProbeTimeout)
defer cancel()
cmd := exec.CommandContext(ctx, "go", "version")
cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local")
out, err := cmd.Output()
if err != nil {
return "", err
}
return parseGoVersion(string(out))
}
func parseGoVersion(out string) (string, error) {
f := strings.Fields(out)
if len(f) < 3 || f[0] != "go" || f[1] != "version" || !strings.HasPrefix(f[2], "go") {
return "", fmt.Errorf("`go version` answered %q", strings.TrimSpace(out))
}
return f[2], nil
}
modules/mesh-build-go/cmd/mesh-build-go/recipe.go
+87
View File
@@ -0,0 +1,87 @@
package main
// The recipe of the Go build module (novox/hq ADR 0405 §1 and §7, ADR 0469 §1, ADR 0470 §4; issue 1483): the one
// Go toolchain version it builds with, the one postgres major version its integration level tests with, and the
// recipe's own version. A node whose installed Go differs from the recipe's refuses every step (guard.go), so all
// builders run this version or none builds.
import (
"crypto/sha256"
"encoding/hex"
"runtime/debug"
"strconv"
"strings"
"github.com/nats-io/nats-server/v2/server"
)
// RecipeGo is the Go version the recipe builds with: the text `go version` prints for it, which is what the guard
// compares and what the indexer reads from <node>.go of the toolchains state.
const RecipeGo = "go1.27.1"
// PostgresMajor is the postgres major version the integration level tests with (ADR 0470 §4).
const PostgresMajor = 18
// RecipeSteps is the recipe's own version: changed whenever the steps of fetch, test or build change in a way
// that can change what they produce (ADR 0405 §7). It is one of the four values the recipe version hashes.
const RecipeSteps = "1"
// natsModule is the library the integration level's bus is started from (ADR 0470 §2).
const natsModule = "github.com/nats-io/nats-server/v2"
// Recipe is every value that makes up the recipe version.
type Recipe struct {
// Steps is RecipeSteps.
Steps string
// Go is the toolchain's version as `go version` prints it.
Go string
// PostgresMajor is the integration level's postgres major version.
PostgresMajor int
// NatsServer is the nats-server library's version, from the program's build information.
NatsServer string
}
// Current is the recipe of this build of the module.
func Current() Recipe {
return Recipe{Steps: RecipeSteps, Go: RecipeGo, PostgresMajor: PostgresMajor, NatsServer: natsServerVersion()}
}
// Version is the recipe version: a hash of the recipe's own version, the Go version, the postgres major version and
// the nats-server version, so a change of any of the four moves it and with it every fingerprint that holds it
// (ADR 0405 §7, ADR 0465 §2). The fields are named in the hashed text, so no two recipes share a text.
func (r Recipe) Version() string {
h := sha256.New()
for _, f := range [][2]string{
{"steps", r.Steps}, {"go", r.Go}, {"postgres", strconv.Itoa(r.PostgresMajor)}, {"nats-server", r.NatsServer},
} {
h.Write([]byte(f[0] + "=" + f[1] + "\n"))
}
return hex.EncodeToString(h.Sum(nil))[:16]
}
// natsServerVersion is the version of the nats-server library this program was built with: its build information
// names it as the module's go.mod does, and has no second source (ADR 0470 §4). A binary built without build
// information (a stripped test binary) falls back to the library's own version constant, which is the same text.
func natsServerVersion() string {
if info, ok := debug.ReadBuildInfo(); ok {
if v := natsVersionOf(info); v != "" {
return v
}
}
return server.VERSION
}
// natsVersionOf is the version build information gives the nats-server library, "v" stripped, or "" when it lists
// none. A replaced library is the replacement's version.
func natsVersionOf(info *debug.BuildInfo) string {
for _, d := range info.Deps {
if d.Path != natsModule {
continue
}
if d.Replace != nil {
d = d.Replace
}
return strings.TrimPrefix(d.Version, "v")
}
return ""
}
modules/mesh-build-go/cmd/mesh-build-go/runner.go
+32
View File
@@ -0,0 +1,32 @@
package main
import (
"context"
"os"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// StepRunner runs the commands of a build step with an environment built from scratch (novox/hq ADR 0466 §4, issue
// 1485): PATH, HOME (the account's own home, where its ssh files and git configuration are) and a fixed locale, plus
// the words the caller passes, and nothing else of this process's environment — which holds the module's bus
// credentials and might hold a password or a token. It is mesh-sdk's clean runner (ExecRunnerClean, issue 1497) with
// that base given; a command is bound by its context alone (the bound 0), so a fetch may outlast a tool call's thirty
// seconds (the step has its own context).
func StepRunner(home string) forgekey.Runner {
run := forgekey.ExecRunnerClean(0)
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
return run(ctx, append(baseEnv(home), env...), name, args...)
}
}
// defaultPath is where a step looks for git, ssh and go when this process was given no PATH.
const defaultPath = "/usr/local/bin:/usr/bin:/bin"
func baseEnv(home string) []string {
path := os.Getenv("PATH")
if path == "" {
path = defaultPath
}
return []string{"PATH=" + path, "HOME=" + home, "LC_ALL=C"}
}
modules/mesh-build-go/cmd/mesh-build-go/store.go
+100
View File
@@ -0,0 +1,100 @@
package main
// The integration level's throwaway store (novox/hq ADR 0470 §1; issue 1486): one postgres server of the installed
// package for each run, `initdb` into the run's own directory, listening on a unix socket in that directory and on no
// TCP address, as the module's account, with durability off as today's check sets it.
import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
// storeStopBound is how long stopping the store is given: it runs after the level, whatever the level did.
const storeStopBound = 30 * time.Second
var postgresVersion = regexp.MustCompile(`PostgreSQL\)\s+(\d+)`)
// postgresMajor is the major version of the installed postgres: `postgres --version` ("postgres (PostgreSQL) 18.6").
func (m *Module) postgresMajor(ctx context.Context) (int, error) {
r := m.Run(ctx, nil, "postgres", "--version")
if r.Status != 0 || r.Err != "" {
return 0, fmt.Errorf("`postgres --version`: %s", firstLine(r.Err+" "+r.Stderr))
}
f := postgresVersion.FindStringSubmatch(r.Stdout)
if f == nil {
return 0, fmt.Errorf("`postgres --version` answered %q", firstLine(r.Stdout))
}
return strconv.Atoi(f[1])
}
// checkPostgres refuses the integration level when the installed postgres major is not the recipe's, naming both
// (ADR 0470 §4); the unit level is not refused. It answers the major it found.
func (m *Module) checkPostgres(ctx context.Context) (int, error) {
major, err := m.postgresMajor(ctx)
if err != nil {
return 0, &buildstep.Refusal{Text: fmt.Sprintf("the installed postgres version could not be read: %v", err), Want: strconv.Itoa(m.Recipe.PostgresMajor)}
}
if major != m.Recipe.PostgresMajor {
return major, &buildstep.Refusal{Text: fmt.Sprintf("postgres %d installed, the integration level is built for postgres %d", major, m.Recipe.PostgresMajor),
Want: strconv.Itoa(m.Recipe.PostgresMajor), Have: strconv.Itoa(major)}
}
return major, nil
}
// socketLimit is the longest path of a socket postgres accepts, with the file's name: it refuses 108 bytes and over.
const socketLimit = 100
// socketName is the file postgres makes in the socket directory.
const socketName = "/.s.PGSQL.5432"
// socketDir makes the directory of the store's socket: under the run when the socket's path stays under socketLimit,
// else a short directory of its own under /tmp (answering short, so the caller removes it). A unix socket's path is
// bounded far below a file's, and a work directory deep in a data root is over it.
func socketDir(run string) (dir string, short bool, err error) {
dir = filepath.Join(run, "pg", "socket")
if len(dir)+len(socketName) < socketLimit {
return dir, false, os.MkdirAll(dir, 0o700)
}
dir, err = os.MkdirTemp("/tmp", "pg")
return dir, true, err
}
// storeSettings are the server's own settings: no TCP address, a socket directory, and no durability.
func storeOptions(socket string) string {
return "-k " + socket + " -c listen_addresses='' -c fsync=off -c synchronous_commit=off -c full_page_writes=off"
}
// startStore makes and starts the run's postgres in run/pg and answers the address tests are given as
// MESH_TEST_POSTGRES, and what stops it. The address names the socket directory percent-encoded, as the controller's
// tests cut the URL at its last slash to name their own databases.
func (m *Module) startStore(ctx context.Context, run string) (string, func(), error) {
data := filepath.Join(run, "pg", "data")
socket, short, err := socketDir(run)
if err != nil {
return "", nil, err
}
stop := func() {
if short {
defer os.RemoveAll(socket)
}
sctx, cancel := context.WithTimeout(context.Background(), storeStopBound)
defer cancel()
m.Run(sctx, nil, "pg_ctl", "-D", data, "-m", "immediate", "-w", "stop")
}
if r := m.Run(ctx, nil, "initdb", "-D", data, "-U", "postgres", "--auth=trust", "-E", "UTF8", "--no-locale"); r.Status != 0 || r.Err != "" {
return "", nil, fmt.Errorf("initdb: %s", firstLine(r.Err+" "+r.Stderr+" "+r.Stdout))
}
if r := m.Run(ctx, nil, "pg_ctl", "-D", data, "-l", filepath.Join(run, "pg", "log"), "-w", "-t", "30", "-o", storeOptions(socket), "start"); r.Status != 0 || r.Err != "" {
stop() // a server that was half started is stopped
return "", nil, fmt.Errorf("pg_ctl start: %s", firstLine(r.Err+" "+r.Stderr+" "+r.Stdout))
}
return "postgres://postgres@/postgres?host=" + url.QueryEscape(socket) + "&sslmode=disable", stop, nil
}
modules/mesh-build-go/cmd/mesh-build-go/test.go
+143
View File
@@ -0,0 +1,143 @@
package main
// The test step (novox/hq ADR 0445; issues 1290 and 1486): a fetched workspace's tests by Go's convention, one level
// at a time, each judged by its own bounds. The unit level is `go test` over every package without the `integration`
// build tag.
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
// testTimeout is go test's own timeout for a level: a bound on a test that never ends, far above any level's bound.
const testTimeout = "10m"
var packageOK = regexp.MustCompile(`^(\./|[A-Za-z0-9])[A-Za-z0-9._/@~-]*(/\.\.\.)?$`)
func (m *Module) testTool() StepTool {
return StepTool{
Name: "build_go_test",
Description: "Run the tests of a fetched workspace by Go's convention, one level: the unit level is go test over every " +
"package without the integration build tag, within five seconds. Answers a step id at once; the step ends on " +
"its key with the level's result: its packages, its times, the tests that failed and the slowest tests. A " +
"level that fails is a finished step whose result says passed false.",
Input: map[string]any{
"type": "object",
"properties": map[string]any{
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
"packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
"description": "package patterns of each module, ./... when none"},
},
"required": []string{"workspace", "level"},
},
Long: true,
Run: m.test,
}
}
// unitBound is the unit level's bound: UnitBound unless a test lowered it.
func (m *Module) unitBound() time.Duration {
if m.UnitBound > 0 {
return m.UnitBound
}
return UnitBound
}
func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
workspace, _ := args["workspace"].(string)
level, _ := args["level"].(string)
packages, err := packagesArg(args["packages"])
if err != nil {
return nil, err
}
if !workspaceOK.MatchString(workspace) {
return nil, fmt.Errorf("%q is not a workspace id", workspace)
}
tree := filepath.Join(m.Work, workspace, "tree")
if _, err := os.Stat(tree); err != nil {
return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
}
switch level {
case LevelUnit:
return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}})
case LevelIntegration:
return m.integration(ctx, workspace, tree, packages)
}
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
}
func packagesArg(v any) ([]string, error) {
if v == nil {
return nil, nil
}
list, ok := v.([]any)
if !ok {
return nil, fmt.Errorf("packages is a list of package patterns")
}
out := []string{}
for _, e := range list {
s, _ := e.(string)
if !packageOK.MatchString(s) || strings.Contains(strings.TrimSuffix(s, "/..."), "..") {
return nil, fmt.Errorf("%q is not a package pattern", s)
}
out = append(out, s)
}
return out, nil
}
// levelOpts are what a level asks of runLevel.
type levelOpts struct {
level string
tags []string
packages []string
bounds Bounds
// env is what the level adds to a go command's environment (MESH_TEST_POSTGRES, MESH_TEST_NATS).
env []string
// setup is the time the level's setup took, which the caller measured; wall says the level's running time is the
// wall time of its go test commands, as the integration level's packages run side by side by go test's own
// parallelism, where the unit level's is the slowest package's.
setup time.Duration
wall bool
// compose is a state of the level to put in the result (the store and the bus it ran against).
store, bus string
}
// runLevel runs `go test -json` in each go.mod directory of the tree and judges the whole against the bounds.
func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestResult, error) {
dirs, err := m.goModDirs(ctx, tree)
if err != nil {
return nil, err
}
res := &TestResult{Level: o.level, Packages: []PackageResult{}, Setup: o.setup.Seconds(), Store: o.store, Bus: o.bus}
packages := o.packages
if len(packages) == 0 {
packages = []string{"./..."}
}
began := m.clock()
for _, d := range dirs {
args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
if len(o.tags) > 0 {
args = append(args, "-tags", strings.Join(o.tags, ","))
}
args = append(args, packages...)
r := m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
rd := newReader(res, d)
noise := rd.read(r.Stdout)
rd.finish()
if (r.Status != 0 || r.Err != "") && len(rd.pkgs) == 0 {
// go test said nothing a test said: it could not run (a go.mod it cannot read, a toolchain it cannot find).
return nil, fmt.Errorf("go test in %s: %s", d, firstLine(r.Err+"\n"+strings.Join(noise, "\n")+"\n"+r.Stderr))
}
}
if o.wall {
res.Elapsed = m.clock().Sub(began).Seconds()
}
res.judge(o.bounds)
return res, nil
}
modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go
+178
View File
@@ -0,0 +1,178 @@
//go:build integration
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
// TestMain makes this test binary the bus when the module starts it as one (bus.go).
func TestMain(m *testing.M) {
if len(os.Args) > 1 && os.Args[1] == busArg {
os.Exit(runBus(os.Args[2:]))
}
os.Exit(m.Run())
}
// needTools skips, loudly, a test whose tools are not installed where the check runs (the mesh's Go toolchain image
// holds no postgres): the level is proven live on a build node, which has them (issue 1486).
func needTools(t *testing.T, what string, tools ...string) {
t.Helper()
for _, tool := range tools {
if _, err := exec.LookPath(tool); err != nil {
t.Skipf("NOT RUN: no %s installed where this check runs (%s); proven live on a build node", tool, what)
}
}
}
// repository copies a fixture repository into a git tree, as a fetch leaves one.
func repository(t *testing.T, name string) string {
t.Helper()
tree := t.TempDir()
if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", name)+"/.", tree).CombinedOutput(); err != nil {
t.Fatalf("%s: %v %s", name, err, out)
}
for _, c := range [][]string{{"init", "-q"}, {"add", "-A"}, {"-c", "user.email=a@b", "-c", "user.name=x", "commit", "-qm", "x"}} {
if out, err := exec.Command("git", append([]string{"-C", tree}, c...)...).CombinedOutput(); err != nil {
t.Fatalf("%s: git %v: %v %s", name, c, err, out)
}
}
return tree
}
// Issue 1290: the unit level against the real go on the fixture repositories in testdata/repos: the tests of
// test_test.go read the output captured from them, and this one proves the capture is what the real go says today. It
// starts go test, so it carries the tag.
func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) {
needTools(t, "the unit level on fixtures", "go", "git")
want := map[string]struct {
passed, noTests bool
failed string
}{
"failing": {failed: "fixture/failing/bad"},
"tagged": {passed: true, noTests: true},
"empty": {passed: true, noTests: true},
"slow": {passed: true},
"broken": {failed: "fixture/broken"},
}
for name, w := range want {
tree := repository(t, name)
home := t.TempDir()
m := &Module{Home: home, Work: t.TempDir(), Cache: t.TempDir()}
m.Run = StepRunner(home)
res, err := m.runLevel(context.Background(), tree, levelOpts{level: LevelUnit, bounds: Bounds{Run: UnitBound}})
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if res.Passed != w.passed || res.NoTests != w.noTests || (w.failed != "" && (len(res.Failed) != 1 || res.Failed[0].Package != w.failed)) {
t.Errorf("%s: result = %+v", name, res)
}
}
}
// processesNamed are the processes whose command line holds text.
func processesNamed(text string) []string {
var out []string
cmdlines, _ := filepath.Glob("/proc/[0-9]*/cmdline")
for _, f := range cmdlines {
if raw, err := os.ReadFile(f); err == nil && strings.Contains(string(raw), text) {
out = append(out, f)
}
}
return out
}
// 1486/AC1: given a fixture repository whose integration test opens MESH_TEST_POSTGRES and MESH_TEST_NATS, the
// integration level passes, and afterwards no postgres or bus process of the run is left and its directory is gone.
// It starts postgres and a bus, so it runs in the merge check and in a try, never on the way.
func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.T) {
needTools(t, "the integration level", "go", "git", "initdb", "pg_ctl", "postgres")
tree := repository(t, "store")
// A short work directory: a unix socket's path is bounded at 108 bytes.
work, err := os.MkdirTemp("/tmp", "w")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(work) })
if err := os.MkdirAll(filepath.Join(work, "ws1"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Rename(tree, filepath.Join(work, "ws1", "tree")); err != nil {
t.Fatal(err)
}
home := t.TempDir()
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
m.Run = StepRunner(home)
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil)
if err != nil {
t.Fatalf("the integration level: %v", err)
}
if !res.Passed || res.Tests != 1 || res.Setup <= 0 || res.Setup > IntegrationSetup.Seconds() {
t.Errorf("result = %+v", res)
}
entries, _ := os.ReadDir(filepath.Join(work, "ws1"))
if len(entries) != 1 || entries[0].Name() != "tree" {
t.Errorf("the workspace holds %v, want only its tree: the run directory is gone", entries)
}
for deadline := time.Now().Add(5 * time.Second); len(processesNamed(work)) > 0 && time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
}
if left := processesNamed(work); len(left) > 0 {
t.Errorf("processes of the run are left: %v", left)
}
}
// 1487/AC2: given a fixture Go module, when it is built twice, both digests are equal and the archive holds one
// executable named as binaryName says. It runs the real go, so it carries the tag.
func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) {
needTools(t, "the build", "go")
work := t.TempDir()
tree := filepath.Join(work, "ws1", "tree")
if err := os.MkdirAll(tree, 0o755); err != nil {
t.Fatal(err)
}
if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", "build")+"/.", tree).CombinedOutput(); err != nil {
t.Fatalf("%v %s", err, out)
}
home := t.TempDir()
m := &Module{Home: home, Work: work, Cache: t.TempDir()}
m.Run = StepRunner(home)
args := map[string]any{"workspace": "ws1", "module": ".", "artifact": "tools"}
first, err := m.build(context.Background(), args)
if err != nil {
t.Fatal(err)
}
second, err := m.build(context.Background(), args)
if err != nil {
t.Fatal(err)
}
a, b := first.(BuildResult), second.(BuildResult)
if a.Digest != b.Digest || a.Binary != "prog-bin" {
t.Errorf("two builds of one tree: %+v and %+v", a, b)
}
raw, err := os.ReadFile(a.Path)
if err != nil {
t.Fatal(err)
}
zr, err := gzip.NewReader(bytes.NewReader(raw))
if err != nil {
t.Fatal(err)
}
tr := tar.NewReader(zr)
h, err := tr.Next()
if err != nil || h.Name != "prog-bin" || h.Mode&0o111 == 0 {
t.Fatalf("first entry = %+v, %v", h, err)
}
if _, err := tr.Next(); err != io.EOF {
t.Errorf("the archive holds more than one file")
}
}
modules/mesh-build-go/cmd/mesh-build-go/test_test.go
+243
View File
@@ -0,0 +1,243 @@
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// capturedRunner answers the commands of a test step from a fixture repository: ls-files with the go.mod, and every
// `go test` with the output captured from the real go (testdata/captured, see its README).
func capturedRunner(t *testing.T, captured string, calls *[]cmdRecord) forgekey.Runner {
t.Helper()
raw, err := os.ReadFile(filepath.Join("testdata", "captured", captured+".json"))
if err != nil {
t.Fatal(err)
}
status := 0
if strings.Contains(string(raw), `"Action":"fail"`) {
status = 1 // go test exits 1 when a test or a build failed
}
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
*calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")})
switch {
case name == "git" && args[2] == "ls-files":
return forgekey.Ran{Stdout: "go.mod\n"}
case name == "go":
return forgekey.Ran{Stdout: string(raw), Status: status}
}
return forgekey.Ran{}
}
}
// workspaceFor is a module whose work directory holds a fetched workspace, and the id of it.
func workspaceFor(t *testing.T, captured string) (*Module, *keptStates, string, *[]cmdRecord) {
t.Helper()
calls := &[]cmdRecord{}
states := newStates()
m := moduleWith(states, "go1.27.1")
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
m.Run = capturedRunner(t, captured, calls)
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
t.Fatal(err)
}
return m, states, "ws1", calls
}
func runUnit(t *testing.T, m *Module, states *keptStates, ws string) (TestResult, delivery.Step) {
t.Helper()
var run func(map[string]any) (any, error)
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == "build_go_test" {
run = tool.Run
}
}
if _, err := run(map[string]any{"workspace": ws, "level": "unit"}); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
var res TestResult
if st.State == delivery.StepPassed {
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatalf("result %s: %v", st.Result, err)
}
}
return res, st
}
// 1290/AC1: a fixture with one passing package and one failing test: the result names the failing test and its
// package, and the level fails. The step itself finished, so it ends passed with the verdict in its result.
func TestUnitLevelNamesTheFailingTestAndFailsTheLevel(t *testing.T) {
m, states, ws, _ := workspaceFor(t, "failing")
res, st := runUnit(t, m, states, ws)
if st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
if res.Passed || res.Level != "unit" {
t.Errorf("result = %+v, want the unit level failed", res)
}
if len(res.Failed) != 1 || res.Failed[0].Package != "fixture/failing/bad" || res.Failed[0].Test != "TestThreeIsFour" ||
!strings.Contains(res.Failed[0].Output, "Three() = 3, want 4") {
t.Errorf("failed = %+v", res.Failed)
}
if !strings.Contains(res.Why, "fixture/failing/bad TestThreeIsFour") {
t.Errorf("why = %q", res.Why)
}
good := false
for _, p := range res.Packages {
if p.Package == "fixture/failing/good" && p.Passed && p.Tests == 1 {
good = true
}
}
if !good || res.Tests != 2 {
t.Errorf("packages = %+v, tests = %d; want the good package passed with its test, 2 tests", res.Packages, res.Tests)
}
}
// 1290/AC2: a fixture whose tests take more than the bound (lowered by the module's option, not by waiting five
// seconds): the level fails naming the bound and the slowest tests; under the real bound the same run passes.
func TestUnitLevelOverItsBoundFailsNamingTheBoundAndTheSlowestTests(t *testing.T) {
m, states, ws, _ := workspaceFor(t, "slow")
m.UnitBound = 10 * time.Millisecond
res, _ := runUnit(t, m, states, ws)
if !res.Passed || !res.OverBound || res.OverWhich != "run" {
t.Fatalf("a level of %.3f s under a bound of %v: passed %v, over_bound %v (%s); the tests passed and the bound was kept no longer", res.Elapsed, m.UnitBound, res.Passed, res.OverBound, res.OverWhich)
}
for _, want := range []string{"over its bound of 0.01 s", "fixture/slow TestTakesAWhile"} {
if !strings.Contains(res.Why, want) {
t.Errorf("why = %q lacks %q", res.Why, want)
}
}
if len(res.Slowest) == 0 || res.Slowest[0].Test != "TestTakesAWhile" || res.Bound != 0.01 {
t.Errorf("slowest = %+v, bound = %v", res.Slowest, res.Bound)
}
m2, states2, ws2, _ := workspaceFor(t, "slow")
if res, _ := runUnit(t, m2, states2, ws2); !res.Passed || res.OverBound || res.Bound != 5 {
t.Errorf("under the bound of five seconds: %+v", res)
}
}
// 1290/AC3: a fixture whose only test carries //go:build integration: go test says [no test files], and the unit
// level reports no tests and passes. No command of the unit level asks for the tag.
func TestUnitLevelOfAFixtureWithOnlyAnIntegrationTestHasNoTests(t *testing.T) {
m, states, ws, calls := workspaceFor(t, "tagged")
res, _ := runUnit(t, m, states, ws)
if !res.Passed || !res.NoTests || res.Tests != 0 || !strings.Contains(res.Why, "no tests") {
t.Errorf("result = %+v, want no tests, passed", res)
}
for _, c := range *calls {
if strings.Contains(c.line, "-tags") {
t.Errorf("the unit level asked for a tag: %s", c.line)
}
}
}
// 1290/AC4: a fixture with no tests at all: the result says the repository has no tests, neither a failure nor a warning.
func TestUnitLevelOfAFixtureWithNoTestsSaysSo(t *testing.T) {
m, states, ws, _ := workspaceFor(t, "empty")
res, st := runUnit(t, m, states, ws)
if st.State != delivery.StepPassed || !res.Passed || !res.NoTests || res.Why != "the repository has no tests at the unit level" || len(res.Failed) != 0 {
t.Errorf("result = %+v (%s)", res, st.State)
}
}
// A package that does not build fails the level with the compiler's words.
func TestUnitLevelOfAPackageThatDoesNotBuildFailsWithTheCompilersLine(t *testing.T) {
m, states, ws, _ := workspaceFor(t, "broken")
res, _ := runUnit(t, m, states, ws)
if res.Passed || len(res.Failed) != 1 || res.Failed[0].Package != "fixture/broken" ||
!strings.Contains(res.Failed[0].Output, `cannot use "one" (untyped string constant) as int value`) {
t.Errorf("result = %+v", res)
}
}
// The command is go test -json in the module's directory of the workspace, with the toolchain pinned to the installed
// one and the module's caches.
func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *testing.T) {
m, states, ws, calls := workspaceFor(t, "empty")
runUnit(t, m, states, ws)
var line string
var env string
for _, c := range *calls {
if strings.HasPrefix(c.line, "go ") {
line, env = c.line, strings.Join(c.env, " ")
}
}
want := "go -C " + filepath.Join(m.Work, ws, "tree", ".") + " test -json -count=1 -timeout 10m ./..."
if line != want {
t.Errorf("command = %q, want %q", line, want)
}
for _, w := range []string{"GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + filepath.Join(m.Cache, "gomod")} {
if !strings.Contains(env, w) {
t.Errorf("environment %s lacks %s", env, w)
}
}
}
// A go test that said nothing a test says (it could not run) ends the step failed, not the level.
func TestAGoTestThatCouldNotRunEndsTheStepFailed(t *testing.T) {
m, states, ws, _ := workspaceFor(t, "empty")
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
if name == "go" {
return forgekey.Ran{Status: 1, Stderr: "go: cannot find main module\n"}
}
return forgekey.Ran{Stdout: "go.mod\n"}
}
_, st := runUnit(t, m, states, ws)
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "go: cannot find main module") {
t.Errorf("the step ended %s: %q", st.State, st.Reason)
}
}
func TestATestOfAnUnknownWorkspaceOrAnotherLevelIsRefused(t *testing.T) {
m, states, _, _ := workspaceFor(t, "empty")
var run func(map[string]any) (any, error)
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == "build_go_test" {
run = tool.Run
}
}
for _, args := range []map[string]any{
{"workspace": "nope", "level": "unit"},
{"workspace": "../ws1", "level": "unit"},
{"workspace": "ws1", "level": "stress"},
{"workspace": "ws1", "level": "unit", "packages": []any{"-run=x"}},
{"workspace": "ws1", "level": "unit", "packages": []any{"../x"}},
} {
run(args)
if st := states.waitEnd(t); st.State != delivery.StepFailed {
t.Errorf("%v: the step ended %s", args, st.State)
}
}
}
// A directory that vendors is tested, and listed, from its vendor directory: fetch downloaded nothing for it, and the
// last value of GOFLAGS is the one go uses.
func TestADirectoryThatVendorsIsReadFromItsVendorDirectory(t *testing.T) {
m, states, ws, calls := workspaceFor(t, "empty")
vendor := filepath.Join(m.Work, ws, "tree", "vendor")
os.MkdirAll(vendor, 0o755)
os.WriteFile(filepath.Join(vendor, "modules.txt"), nil, 0o644)
runUnit(t, m, states, ws)
flags := ""
for _, c := range *calls {
if strings.HasPrefix(c.line, "go ") {
for _, w := range c.env {
if strings.HasPrefix(w, "GOFLAGS=") {
flags = w
}
}
}
}
if flags != "GOFLAGS=-mod=vendor" {
t.Errorf("the last GOFLAGS of go test in a vendoring directory is %q", flags)
}
}
modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/README
Vendored
+10
View File
@@ -0,0 +1,10 @@
Each file is the standard output of `GOTOOLCHAIN=local GOFLAGS=-mod=mod go test -count=1 -json ./...` (go1.27.1)
run in the fixture module of the same name under ../repos, captured on 2026-10-20 for issue 1290. The unit
tests feed them to the parser through a fake runner, so no test here runs `go test`.
golist-external.json is `GOTOOLCHAIN=local GOFLAGS=-mod=mod go list -json=ImportPath,Standard,DepOnly,Module
git.novox.be/novox/mesh-sdk/go/buildstep` run in this module (go1.27.1, 2026-10-20 for issue 1291): a package of a module
outside the tree as go list describes it.
toolchain-go-entry.txt is lines 155-191 of mesh-control's internal/builder/toolchain.go at origin/main 073ccc29 (fetched
2026-10-20): the Go entry of today's build agent's recipe, which the build step's flags are compared with.
modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/broken.json
Vendored
+6
View File
@@ -0,0 +1,6 @@
{"ImportPath":"fixture/broken [fixture/broken.test]","Action":"build-output","Output":"# fixture/broken [fixture/broken.test]\n"}
{"ImportPath":"fixture/broken [fixture/broken.test]","Action":"build-output","Output":"./broken.go:3:25: cannot use \"one\" (untyped string constant) as int value in return statement\n"}
{"ImportPath":"fixture/broken [fixture/broken.test]","Action":"build-fail"}
{"Time":"2026-10-20T22:20:06.344233803+02:00","Action":"start","Package":"fixture/broken"}
{"Time":"2026-10-20T22:20:06.344344815+02:00","Action":"output","Package":"fixture/broken","Output":"FAIL\tfixture/broken [build failed]\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.344358865+02:00","Action":"fail","Package":"fixture/broken","Elapsed":0,"FailedBuild":"fixture/broken [fixture/broken.test]"}
modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/empty.json
Vendored
+3
View File
@@ -0,0 +1,3 @@
{"Time":"2026-10-20T22:20:06.064193534+02:00","Action":"start","Package":"fixture/empty"}
{"Time":"2026-10-20T22:20:06.064489419+02:00","Action":"output","Package":"fixture/empty","Output":"? \tfixture/empty\t[no test files]\n"}
{"Time":"2026-10-20T22:20:06.064510249+02:00","Action":"skip","Package":"fixture/empty","Elapsed":0}
modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/failing.json
Vendored
+17
View File
@@ -0,0 +1,17 @@
{"Time":"2026-10-20T22:20:06.028267402+02:00","Action":"start","Package":"fixture/failing/bad"}
{"Time":"2026-10-20T22:20:06.028528997+02:00","Action":"start","Package":"fixture/failing/good"}
{"Time":"2026-10-20T22:20:06.030412986+02:00","Action":"run","Package":"fixture/failing/bad","Test":"TestThreeIsFour"}
{"Time":"2026-10-20T22:20:06.030453876+02:00","Action":"output","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Output":"=== RUN TestThreeIsFour\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.030494057+02:00","Action":"output","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Output":" bad_test.go:7: Three() = 3, want 4\n","OutputType":"error"}
{"Time":"2026-10-20T22:20:06.030511077+02:00","Action":"output","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Output":"--- FAIL: TestThreeIsFour (0.00s)\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.030519017+02:00","Action":"fail","Package":"fixture/failing/bad","Test":"TestThreeIsFour","Elapsed":0}
{"Time":"2026-10-20T22:20:06.030529158+02:00","Action":"output","Package":"fixture/failing/bad","Output":"FAIL\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.030546038+02:00","Action":"run","Package":"fixture/failing/good","Test":"TestTwo"}
{"Time":"2026-10-20T22:20:06.030561528+02:00","Action":"output","Package":"fixture/failing/good","Test":"TestTwo","Output":"=== RUN TestTwo\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.030570028+02:00","Action":"output","Package":"fixture/failing/good","Test":"TestTwo","Output":"--- PASS: TestTwo (0.00s)\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.030577128+02:00","Action":"pass","Package":"fixture/failing/good","Test":"TestTwo","Elapsed":0}
{"Time":"2026-10-20T22:20:06.030583418+02:00","Action":"output","Package":"fixture/failing/good","Output":"PASS\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.030962814+02:00","Action":"output","Package":"fixture/failing/bad","Output":"FAIL\tfixture/failing/bad\t0.002s\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.030974515+02:00","Action":"output","Package":"fixture/failing/good","Output":"ok \tfixture/failing/good\t0.002s\n"}
{"Time":"2026-10-20T22:20:06.030991305+02:00","Action":"pass","Package":"fixture/failing/good","Elapsed":0.002}
{"Time":"2026-10-20T22:20:06.030982975+02:00","Action":"fail","Package":"fixture/failing/bad","Elapsed":0.003}
modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/golist-external.json
Vendored
+13
View File
@@ -0,0 +1,13 @@
{
"ImportPath": "git.novox.be/novox/mesh-sdk/go/buildstep",
"Module": {
"Path": "git.novox.be/novox/mesh-sdk/go",
"Version": "v0.1.14-0.20261020200715-9e296295c05e",
"Time": "2026-10-20T20:07:15Z",
"Dir": "/home/jochen/go/pkg/mod/git.novox.be/novox/mesh-sdk/go@v0.1.14-0.20261020200715-9e296295c05e",
"GoMod": "/home/jochen/go/pkg/mod/cache/download/git.novox.be/novox/mesh-sdk/go/@v/v0.1.14-0.20261020200715-9e296295c05e.mod",
"GoVersion": "1.26.0",
"Sum": "h1:uf2o2A6dBOrISDUVw6U7+fv8ylwwF+UfRrGWKIQMFOE=",
"GoModSum": "h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY="
}
}
modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/slow.json
Vendored
+12
View File
@@ -0,0 +1,12 @@
{"Time":"2026-10-20T22:20:06.21802535+02:00","Action":"start","Package":"fixture/slow"}
{"Time":"2026-10-20T22:20:06.222338257+02:00","Action":"run","Package":"fixture/slow","Test":"TestTakesAWhile"}
{"Time":"2026-10-20T22:20:06.222387677+02:00","Action":"output","Package":"fixture/slow","Test":"TestTakesAWhile","Output":"=== RUN TestTakesAWhile\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.282860204+02:00","Action":"output","Package":"fixture/slow","Test":"TestTakesAWhile","Output":"--- PASS: TestTakesAWhile (0.06s)\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.282917774+02:00","Action":"pass","Package":"fixture/slow","Test":"TestTakesAWhile","Elapsed":0.06}
{"Time":"2026-10-20T22:20:06.282937215+02:00","Action":"run","Package":"fixture/slow","Test":"TestQuick"}
{"Time":"2026-10-20T22:20:06.282943535+02:00","Action":"output","Package":"fixture/slow","Test":"TestQuick","Output":"=== RUN TestQuick\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.282955095+02:00","Action":"output","Package":"fixture/slow","Test":"TestQuick","Output":"--- PASS: TestQuick (0.00s)\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.282963365+02:00","Action":"pass","Package":"fixture/slow","Test":"TestQuick","Elapsed":0}
{"Time":"2026-10-20T22:20:06.282971185+02:00","Action":"output","Package":"fixture/slow","Output":"PASS\n","OutputType":"frame"}
{"Time":"2026-10-20T22:20:06.283463443+02:00","Action":"output","Package":"fixture/slow","Output":"ok \tfixture/slow\t0.063s\n"}
{"Time":"2026-10-20T22:20:06.283481213+02:00","Action":"pass","Package":"fixture/slow","Elapsed":0.065}
modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/tagged.json
Vendored
+3
View File
@@ -0,0 +1,3 @@
{"Time":"2026-10-20T22:20:06.049418313+02:00","Action":"start","Package":"fixture/tagged"}
{"Time":"2026-10-20T22:20:06.049698258+02:00","Action":"output","Package":"fixture/tagged","Output":"? \tfixture/tagged\t[no test files]\n"}
{"Time":"2026-10-20T22:20:06.049730188+02:00","Action":"skip","Package":"fixture/tagged","Elapsed":0}
modules/mesh-build-go/cmd/mesh-build-go/testdata/captured/toolchain-go-entry.txt
Vendored
+37
View File
@@ -0,0 +1,37 @@
Language: "go",
Base: "mesh-tools-go",
Artifact: "build",
// **The mesh's own components, and not modules.** The warning above this list — that every
// language is another implementation of the contracts modules share, so adding one commits
// to keeping N implementations in step — does not attach here. Go is how the host, the
// control plane and the builder are written, and none of them is a module in that sense:
// the host is what APPLIES modules. So there is no SDK obligation, and the reason this
// entry did not exist was that nothing needed to compile the mesh itself
// (novox/hq ADR 0142, and 04-ISSUES/142 where that is why nothing delivers the host).
//
// Static, because what a machine ends up holding is a file rather than a container, and a
// binary that needs a libc it did not bring is a delivery that works until a machine
// differs. Trimmed of its own paths for the same reason a version comes from where it sits
// rather than from the linker: two builds of one commit produce the same bytes.
//
// **And of one build source at two commits** (novox/hq ADR 0267 rule 6, issue 363): a module built
// from its repository's root is compiled with the clone's `.git` in view, and Go stamps the commit,
// its time and whether the tree was modified into the binary unless told not to — so a merge that
// changed only a README made a new digest of an unchanged program.
Compile: []string{
"env", "CGO_ENABLED=0", "GOFLAGS=-trimpath -buildvcs=false",
"go", "build",
},
// Stripped of symbols and debug info: what a machine holds is a file it runs, not one it
// debugs, and the difference measured 12.2MB against 8.5MB.
LinkerFlags: []string{"-s", "-w"},
OutputFlag: "-o",
// Pointed at the package the artifact is built `from`, compiled whole. Go writes the binary
// into the output directory, named after the package — so the bundle a machine unpacks is a
// directory holding one executable, which is what the delivery mechanism expects
// (novox/hq ADR 0141).
Unit: UnitPackage,
// The mesh's own Go components read the system they were built for from this variable, and
// refuse to touch a machine without one.
SystemStamp: "main.builtFor",
},
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/broken.go
Vendored
+3
View File
@@ -0,0 +1,3 @@
package broken
func One() int { return "one" }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/broken_test.go
Vendored
+5
View File
@@ -0,0 +1,5 @@
package broken
import "testing"
func TestOne(t *testing.T) { One() }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/broken/go.mod
Vendored
+3
View File
@@ -0,0 +1,3 @@
module fixture/broken
go 1.25.0
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/cmd/prog/main.go
Vendored
+9
View File
@@ -0,0 +1,9 @@
// Command prog is what the build step of issue 1487 builds in its integration test.
package main
import "fmt"
// builtFor is stamped by the linker.
var builtFor string
func main() { fmt.Println("prog built for", builtFor) }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/go.mod
Vendored
+3
View File
@@ -0,0 +1,3 @@
module fixture/build
go 1.25.0
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/build/module.json
Vendored
+17
View File
@@ -0,0 +1,17 @@
{
"module": "fixture-build",
"version": "1",
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/prog",
"binary": "prog-bin",
"loads": ["prog-bin"]
}
]
}
}
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/a.go
Vendored
+4
View File
@@ -0,0 +1,4 @@
package a
// A is what a's tests check.
func A() int { return 1 }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/a_test.go
Vendored
+15
View File
@@ -0,0 +1,15 @@
package a
import (
"os"
"testing"
"fixture/digest/b"
)
func TestAReadsBAndItsTestdata(t *testing.T) {
raw, err := os.ReadFile("testdata/x.txt")
if err != nil || A()+b.B() != 3 || len(raw) == 0 {
t.Fatal("a")
}
}
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/a/testdata/x.txt
Vendored
+1
View File
@@ -0,0 +1 @@
x
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/b/b.go
Vendored
+4
View File
@@ -0,0 +1,4 @@
package b
// B is reached by a's tests.
func B() int { return 2 }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/c.go
Vendored
+6
View File
@@ -0,0 +1,6 @@
package c
import _ "embed"
//go:embed note.txt
var Note string
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/c_test.go
Vendored
+9
View File
@@ -0,0 +1,9 @@
package c
import "testing"
func TestCHasANote(t *testing.T) {
if Note == "" {
t.Fatal("c")
}
}
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/c/note.txt
Vendored
+1
View File
@@ -0,0 +1 @@
note
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/d/d.go
Vendored
+4
View File
@@ -0,0 +1,4 @@
package d
// D is reached by no test.
func D() int { return 4 }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/digest/go.mod
Vendored
+3
View File
@@ -0,0 +1,3 @@
module fixture/digest
go 1.25.0
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/empty/empty.go
Vendored
+3
View File
@@ -0,0 +1,3 @@
package empty
func Nothing() {}
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/empty/go.mod
Vendored
+3
View File
@@ -0,0 +1,3 @@
module fixture/empty
go 1.25.0
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/bad/bad.go
Vendored
+3
View File
@@ -0,0 +1,3 @@
package bad
func Three() int { return 3 }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/bad/bad_test.go
Vendored
+9
View File
@@ -0,0 +1,9 @@
package bad
import "testing"
func TestThreeIsFour(t *testing.T) {
if Three() != 4 {
t.Fatalf("Three() = %d, want 4", Three())
}
}
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/go.mod
Vendored
+3
View File
@@ -0,0 +1,3 @@
module fixture/failing
go 1.25.0
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/good/good.go
Vendored
+3
View File
@@ -0,0 +1,3 @@
package good
func Two() int { return 2 }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/failing/good/good_test.go
Vendored
+9
View File
@@ -0,0 +1,9 @@
package good
import "testing"
func TestTwo(t *testing.T) {
if Two() != 2 {
t.Fatal("two")
}
}
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/go.mod
Vendored
+3
View File
@@ -0,0 +1,3 @@
module fixture/slow
go 1.25.0
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/slow.go
Vendored
+3
View File
@@ -0,0 +1,3 @@
package slow
func Zero() int { return 0 }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/slow/slow_test.go
Vendored
+16
View File
@@ -0,0 +1,16 @@
package slow
import (
"testing"
"time"
)
func TestTakesAWhile(t *testing.T) {
time.Sleep(60 * time.Millisecond)
}
func TestQuick(t *testing.T) {
if Zero() != 0 {
t.Fatal("zero")
}
}
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/go.mod
Vendored
+3
View File
@@ -0,0 +1,3 @@
module fixture/store
go 1.25.0
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/store.go
Vendored
+4
View File
@@ -0,0 +1,4 @@
package store
// Name is what the integration test below finds a store and a bus for.
const Name = "store"
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/store/store_integration_test.go
Vendored
+37
View File
@@ -0,0 +1,37 @@
//go:build integration
package store
import (
"bufio"
"net"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// The fixture of issue 1486: the store and the bus a run gives its tests are there, reached as a test reaches them.
func TestTheStoreAndTheBusAreThere(t *testing.T) {
u, err := url.Parse(os.Getenv("MESH_TEST_POSTGRES"))
if err != nil || u.Query().Get("host") == "" {
t.Fatalf("MESH_TEST_POSTGRES = %q", os.Getenv("MESH_TEST_POSTGRES"))
}
pg, err := net.DialTimeout("unix", filepath.Join(u.Query().Get("host"), ".s.PGSQL.5432"), 5*time.Second)
if err != nil {
t.Fatalf("the store's socket: %v", err)
}
pg.Close()
bus, err := net.DialTimeout("tcp", strings.TrimPrefix(os.Getenv("MESH_TEST_NATS"), "nats://"), 5*time.Second)
if err != nil {
t.Fatalf("the bus: %v", err)
}
defer bus.Close()
line, err := bufio.NewReader(bus).ReadString('\n')
if err != nil || !strings.HasPrefix(line, "INFO ") || !strings.Contains(line, `"jetstream":true`) {
t.Fatalf("the bus said %q (%v), want an INFO line with JetStream on", line, err)
}
}
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/go.mod
Vendored
+3
View File
@@ -0,0 +1,3 @@
module fixture/tagged
go 1.25.0
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/tagged.go
Vendored
+3
View File
@@ -0,0 +1,3 @@
package tagged
func One() int { return 1 }
modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/tagged/tagged_integration_test.go
Vendored
+11
View File
@@ -0,0 +1,11 @@
//go:build integration
package tagged
import "testing"
func TestNeedsAStore(t *testing.T) {
if One() != 1 {
t.Fatal("one")
}
}
modules/mesh-build-go/cmd/mesh-build-go/tools.go
+53
View File
@@ -0,0 +1,53 @@
package main
import (
"context"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The tools this module serves. A tool is named as the module is: build_go_ and its verb (a tool name has no dot).
const (
StatusTool = "build_go_status"
HealthTool = "build_go_health"
)
// StepTool is one step tool of the module: fetch, test, digest or build. Long is a step that answers at once and
// ends on its key (ADR 0469 §2); the others answer in their call. Both run behind the toolchain guard.
type StepTool struct {
Name string
Description string
Input map[string]any
Long bool
Run func(ctx context.Context, args map[string]any) (any, error)
}
// StepTools are the module's step tools. Each step's issue adds its own.
func (m *Module) StepTools() []StepTool {
return []StepTool{m.fetchTool(), m.testTool(), m.digestTool(), m.buildTool()}
}
// Tools is every tool the module serves: its step tools behind the guard, its status and its health.
func (m *Module) Tools(steps []StepTool) []stdio.Tool {
tools := []stdio.Tool{}
for _, s := range steps {
t := stdio.Tool{Name: s.Name, Description: s.Description, Input: s.Input}
if s.Long {
t.Run = m.Long(s.Name, s.Run)
} else {
t.Run = m.Short(s.Run)
}
tools = append(tools, t)
}
return append(tools,
stdio.Tool{Name: StatusTool,
Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " +
"the recipe version, the postgres major version and the nats-server version the integration level uses, and " +
"why it refuses to build if it does. Read-only.",
Run: func(map[string]any) (any, error) { return m.Status(m.Ctx), nil }},
stdio.Tool{Name: HealthTool,
Description: "How the Go build module is: unhealthy while the Go installed on this machine is not the version its " +
"recipe builds with, saying both versions; healthy otherwise. Read-only.",
Run: func(map[string]any) (any, error) { return m.Health(m.Ctx), nil }},
)
}
modules/mesh-build-go/files/forge
+7
View File
@@ -0,0 +1,7 @@
Host ${bound:forge-account:ssh_alias}
HostName ${bound:forge-account:ssh_host}
Port ${bound:forge-account:ssh_port}
HostKeyAlias ${bound:forge-account:ssh_alias}
User git
IdentityFile ~/.ssh/mesh_ed25519
IdentitiesOnly yes
modules/mesh-build-go/files/gitconfig
+2
View File
@@ -0,0 +1,2 @@
[url "ssh://${bound:forge-account:ssh_alias}/"]
insteadOf = ${bound:forge-account:https_base}
modules/mesh-build-go/files/known_hosts
+1
View File
@@ -0,0 +1 @@
${bound:forge-account:ssh_alias} ${bound:forge-account:host_key}
modules/mesh-build-go/go.mod
+21
View File
@@ -0,0 +1,21 @@
module meshbuildgo
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d
github.com/nats-io/nats-server/v2 v2.11.17
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
modules/mesh-build-go/go.sum
+31
View File
@@ -0,0 +1,31 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020200715-9e296295c05e h1:uf2o2A6dBOrISDUVw6U7+fv8ylwwF+UfRrGWKIQMFOE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020200715-9e296295c05e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020203945-1572e50e0ef0 h1:mr9HyARwzeZT7FxFHv1Xc2z7QErvDZ/F/pBMIpjGSd0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020203945-1572e50e0ef0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d h1:mmOCJphjQN15jNVh+sMQqnmJ3UiYltiuQRg/zw3wFtw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020204626-36d6f7745c8d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
modules/mesh-build-go/module.json
+148
View File
@@ -0,0 +1,148 @@
{
"module": "mesh-build-go",
"version": "1",
"runs-as": "mesh-build-go",
"capabilities": [
"package-manager",
"user-namespaces"
],
"state": [
"toolchains",
"keys",
"steps"
],
"tools": [
"build_go_fetch",
"build_go_test",
"build_go_digest",
"build_go_build",
"build_go_status",
"build_go_health"
],
"streams": [
{
"name": "toolchains-go",
"events": ["toolchain.**"]
}
],
"requires": [
"forge-account"
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-build-go",
"runtime-health": {
"kind": "tool",
"tool": "build_go_health",
"interval": "60s",
"timeout": "10s",
"looks": 2,
"grace": "60s"
},
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-build-go",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-build-go"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-build-go",
"place": ".",
"unknown": "program"
},
{
"id": "work",
"type": "directory",
"path": "/var/lib/mesh-build-go-work",
"mode": "0700",
"owner": "mesh-build-go",
"unknown": "program"
},
{
"id": "cache",
"type": "directory",
"path": "/var/lib/mesh-build-go-cache",
"mode": "0700",
"owner": "mesh-build-go",
"unknown": "program"
},
{
"id": "ssh-dir",
"type": "directory",
"path": "${dir:state}/.ssh",
"mode": "0700",
"owner": "mesh-build-go",
"unknown": "program"
},
{
"id": "ssh-config",
"type": "file",
"path": "${dir:ssh-dir}/config",
"owner": "mesh-build-go",
"mode": "0600",
"template": "files/forge"
},
{
"id": "known-hosts",
"type": "file",
"path": "${dir:ssh-dir}/known_hosts",
"owner": "mesh-build-go",
"mode": "0644",
"template": "files/known_hosts"
},
{
"id": "git-config",
"type": "file",
"path": "${dir:state}/.config/git/config",
"owner": "mesh-build-go",
"mode": "0644",
"template": "files/gitconfig"
},
{
"id": "go",
"type": "package",
"package": "go"
},
{
"id": "git",
"type": "package",
"package": "git"
},
{
"id": "openssh",
"type": "package",
"package": "openssh"
},
{
"id": "postgresql",
"type": "package",
"package": "postgresql"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-build-go",
"binary": "mesh-build-go",
"loads": [
"mesh-build-go"
]
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-build-node/README.md
+67
View File
@@ -0,0 +1,67 @@
# mesh-build-node — the Node and TypeScript build module
Design 47, release R4c-3 (novox/hq ADR 0405, ADR 0466 §1, ADR 0469 §1 and §4). The module that builds and checks
TypeScript code on a build machine, as an account of its own (`runs-as`, no login shell, home at mode 0700), with the
packages it declares (`nodejs`, `npm`, `git`, `openssh`) and the capability `user-namespaces`. The bundle is Go
(`cmd/mesh-build-node`). It claims no seat: it is assigned to the build machines (ADR 0469 §6) and runs dark until the
builder uses it. Its forge key and ssh files are the same as mesh-build-go's (issue 1484).
## The recipe
`recipe.go` names the one Node version it builds with (`RecipeNode`, `v26.10.0`, as `node --version` prints it) and the
recipe's own version (`RecipeSteps`). The recipe version is a hash of those two and of the bytes of the lock file
`toolchain/package-lock.json`, so one changed byte of the lock, another Node or another recipe moves it.
## The toolchain directory
`toolchain/package.json` and its committed `package-lock.json` name typescript 5.9.3, esbuild 0.25.12, @types/node 22.20.1 (the types tsc checks node imports against),
@novox/mesh-sdk 0.1.14 and nats 2.29.3. The mesh unpacks both, as the archive artifact `toolchain`, into `<home>/toolchain/` (resource `toolchain-dir`; a
manifest names no host, and the lock does, so it is not a template); the program reads the lock from there. When `<cache dir>/toolchain/<recipe version>` is missing
the module runs `npm ci --ignore-scripts` there once (a directory without its `.installed` mark counts as missing).
**How the registry is reached** (the choice of issue 1488): the lock holds @novox/mesh-sdk at the forge's public
address, and npm 12 installs a locked tarball only from the registry it is configured with. The module therefore
requires `npm-package-registry` (read access, provided by gitea) and, like the lab module, takes its binding and sealed
secret as files in its home. The install gets a private npm config file with the `@novox` scope at the binding's
`public-root` (else scheme, address, port and npm path) and the bound account's credential, and removes it afterwards;
every other package resolves from the public registry. This is the way today's build agent names it.
The runtime tree for bundles with externals (the pruned sdk and nats today in `/app/runtime`) is not installed
separately: sdk and nats are in this directory's `node_modules`, and the build step (issue 1490) takes them from there.
## The guard and the health
At start, every ten minutes, before every step and at every health look, the module reads the lock again and the
installed Node (`node --version`) and writes it with the recipe version to `<node>.typescript` of its `toolchains`
state. A machine whose Node is not the recipe's refuses every step, the step ending `refused` on its key in `steps` with
both versions named, and `build_node_health` (the module's `runtime-health`) answers unhealthy with the same text, so
the controller raises the machine's condition. A step runs only after the toolchain directory is installed.
## Tools
`build_node_status` and `build_node_health` answer in their call. The step tools run behind the guard and the
toolchain install: a long step answers at once with its id and ends on `<node>.<id>` of `steps` (ADR 0469 §2).
- `build_node_fetch` {repository, commit, ref} (issue 1489): the commit over ssh as the module's account
(`buildstep.FetchCommit`, remote `ssh://mesh-forge/<owner>/<name>.git`) into `<work>/<id>/tree`, then for each
TypeScript module (a module.json with a typescript artifact) the packages its package.json names beyond the SDK,
`typescript` and `@types/*` (the toolchain carries those): none installs nothing; some need a package-lock.json, and
are installed as the build agent does (scratch copy without the SDK,
devDependencies and scripts; `npm ci --omit=dev --omit=peer --ignore-scripts --no-bin-links`; copies of the SDK
removed). The mesh's registry reaches `npm ci` through a private config file (`--userconfig`), so
other `@novox/` packages install. A module with packages and no lock does not fail the fetch: its result lists it
under `refused` with the reason, and a build of that bundle fails naming it (issue 1459). Every command runs through `forgekey.ExecRunnerClean(0)` with PATH, HOME, the locale and its own
words, nothing of this process's environment. Workspaces older than a day are removed at each fetch.
- `build_node_test` {workspace, module, level} (issue 1491): the unit level is the type-check (the toolchain's tsc, the
package's `tsconfig.json` or the build agent's flags over its .ts files) and the `test` script; the integration level
is the `test:integration` script, the type-check being its setup. Bounds (the Go module's; ADR 0445 names none for
TypeScript): unit 5 s, integration 30 s setup and 60 s running. The test script finds `tsc` and `esbuild` on its PATH through `<work>/<id>/bin` (shell scripts calling the toolchain's). A level over its bound says `over_bound` and passes;
a failed one finishes the step with `passed` false and the output (the step fails only when the tests could not be run). Neither script nor .ts files: no tests. The toolchain makes no bin
links, so a script runs `node` and files, not `tsc` by name.
- `build_node_build` {workspace, module, artifact} (issue 1490): the build agent's tsc and esbuild lines
(mesh-controller internal/builder, cited in `build.go`), the launchers, `package.json`, `buildstep.Pack`, and the
digest. No network. The build runs in a staging copy of the module at `<work>/<id>/stage-<artifact>/modules/module`, with the
toolchain's packages (not typescript or esbuild) copied to `stage-<artifact>/node_modules`: the build agent's depth
(`/app/modules/module`, `/app/node_modules`), so esbuild's `../../node_modules/...` path comments, and the bundle's
digest, equal its. Copies, not links, because the mesh makes no symlink. A bundle that keeps an external gets them in its own `node_modules`.
The packed archive is written beside the tree as `<artifact>.tar.gz`.
modules/mesh-build-node/cmd/mesh-build-node/build.go
+283
View File
@@ -0,0 +1,283 @@
package main
// The build step (novox/hq ADR 0405 §3 and §7, ADR 0469 §4; issue 1490): a TypeScript bundle of a fetched workspace
// compiled with the toolchain's tsc, bundled with its esbuild, given its launchers, packed with mesh-sdk's Pack and
// ended with the archive's digest. No network. The commands and flags are the build agent's, copied from
// mesh-controller origin/main internal/builder: toolchain.go:143-147 (tsc), builder.go:1637-1705 (compile),
// :1926-1956 (writeLaunchers), :1971-2039 (bundled), :1735-1747 (sourcesFor). Where they ran in the toolchain image
// they run here from the module's directory with the toolchain's real paths, and tsc and esbuild find the SDK in the
// workspace's node_modules (workspace.go).
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
// launcherSuffix is what an entrypoint's launcher is called beside it: index.js is started as index.serve.mjs
// (novox/hq ADR 0193).
const launcherSuffix = ".serve.mjs"
// Out is where a bundle's compiled files go in the module's directory (builder toolchain.go:118).
func Out(artifact string) string { return ".mesh-build/" + artifact }
// banner is esbuild's createRequire shim: CommonJS inlined into an ES module still finds `require`.
const banner = "--banner:js=import { createRequire as __meshRequire } from 'node:module'; const require = __meshRequire(import.meta.url);"
// artifact is the part of a module.json artifact the build reads.
type artifact struct {
Name string `json:"name"`
Language string `json:"language"`
Entrypoints []string `json:"entrypoints"`
External []string `json:"external"`
}
// BuildResult is what a passed build step says.
type BuildResult struct {
Workspace string `json:"workspace"`
Module string `json:"module"`
Artifact string `json:"artifact"`
// Digest is "sha256:" and the hex of the archive's bytes (buildstep.Pack).
Digest string `json:"digest"`
Bytes int `json:"bytes"`
// Archive is the packed file, in the workspace beside its tree.
Archive string `json:"archive"`
Files []string `json:"files"`
}
func (m *Module) buildTool() StepTool {
return StepTool{
Name: "build_node_build",
Description: "Build a TypeScript bundle of a fetched workspace: compile it with the toolchain's tsc, bundle each " +
"entrypoint with its esbuild, write the launchers and package.json, pack the bundle and answer its digest. " +
"Answers a step id at once; the step ends on its key in the steps state with the digest. No network.",
Input: map[string]any{
"type": "object",
"properties": map[string]any{
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch step answered"},
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, '.' when it is the root"},
"artifact": map[string]any{"type": "string", "description": "the name of the bundle in the module's module.json"},
},
"required": []string{"workspace", "artifact"},
},
Long: true,
Run: m.build,
}
}
func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
workspace, _ := args["workspace"].(string)
module, _ := args["module"].(string)
name, _ := args["artifact"].(string)
root, dir, err := m.moduleDirOf(workspace, module)
if err != nil {
return nil, err
}
a, err := readArtifact(dir, name)
if err != nil {
return nil, err
}
if why, err := lockProblem(dir, module); err != nil || why != "" {
if err == nil {
err = fmt.Errorf("%s was refused by the fetch: %s", name, why)
}
return nil, err
}
// Built at the build agent's depth (builder.go: the module at /app/modules/module, the packages at /app/node_modules),
// so the path comments esbuild writes, `../../node_modules/...`, and with them the bundle's bytes equal its.
stage := filepath.Join(root, "stage-"+a.Name)
if err := os.RemoveAll(stage); err != nil {
return nil, err
}
src := dir
dir = filepath.Join(stage, "modules", "module")
if err := copyTree(src, dir); err != nil {
return nil, fmt.Errorf("staging the module: %w", err)
}
if err := copyPackages(filepath.Join(m.toolchainDir(), "node_modules"), filepath.Join(stage, "node_modules")); err != nil {
return nil, fmt.Errorf("providing the toolchain's packages to the build: %w", err)
}
out, final := Out(a.Name), Out(a.Name)+".bundled"
if err := os.MkdirAll(filepath.Join(dir, out), 0o755); err != nil {
return nil, err
}
// tsc: builder.go:1637-1705 with toolchain.go:143-147's flags, rooted at the module so an entrypoint lands where
// it is named.
tsc := []string{m.tsc(), "--module", "NodeNext", "--moduleResolution", "NodeNext", "--target", "ES2022",
"--rootDir", ".", "--outDir", out}
tsc = append(tsc, sourcesFor(a.Entrypoints, out, ".ts")...)
if why := ranFailed(m.runIn(ctx, dir, nil, "node", tsc...)); why != "" {
return nil, fmt.Errorf("tsc for %s: %s", a.Name, why)
}
launchers, err := writeLaunchers(filepath.Join(dir, out), a)
if err != nil {
return nil, err
}
if err := m.bundle(ctx, dir, a, launchers); err != nil {
return nil, err
}
bundleDir := filepath.Join(dir, final)
if len(a.External) > 0 {
if err := copyPackages(filepath.Join(m.toolchainDir(), "node_modules"), filepath.Join(bundleDir, "node_modules")); err != nil {
return nil, fmt.Errorf("copying the packages %s keeps external: %w", a.Name, err)
}
}
archive, digest, err := buildstep.Pack(bundleDir)
if err != nil {
return nil, fmt.Errorf("packing %s: %w", a.Name, err)
}
file := filepath.Join(root, a.Name+".tar.gz")
if err := os.WriteFile(file, archive, 0o600); err != nil {
return nil, err
}
files := []string{}
_ = filepath.WalkDir(bundleDir, func(p string, d os.DirEntry, err error) error {
if err == nil && !d.IsDir() {
rel, _ := filepath.Rel(bundleDir, p)
files = append(files, filepath.ToSlash(rel))
}
return nil
})
sort.Strings(files)
return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive), Archive: file, Files: files}, nil
}
// bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is
// bundled in place under its own name, a launcher keeps its name with the extension .mjs.
func (m *Module) bundle(ctx context.Context, dir string, a artifact, launchers map[string]string) error {
out, final := Out(a.Name), Out(a.Name)+".bundled"
if err := os.RemoveAll(filepath.Join(dir, final)); err != nil {
return err
}
if err := os.MkdirAll(filepath.Join(dir, final), 0o755); err != nil {
return err
}
common := []string{"--bundle", "--platform=node", "--format=esm", "--target=node22",
"--outbase=" + out, "--outdir=" + final, "--log-level=warning", banner}
for _, x := range a.External {
common = append(common, "--external:"+x)
}
var plain, launch []string
for _, e := range a.Entrypoints {
if strings.HasSuffix(e, ".js") {
plain = append(plain, out+"/"+e)
}
}
for _, l := range sortedValues(launchers) {
launch = append(launch, out+"/"+l)
}
step := func(entries []string, extra ...string) error {
if len(entries) == 0 {
return nil
}
args := append(append(append([]string{}, entries...), common...), extra...)
if why := ranFailed(m.runIn(ctx, dir, nil, m.esbuild(), args...)); why != "" {
return fmt.Errorf("esbuild for %s: %s", a.Name, why)
}
return nil
}
if err := step(plain); err != nil {
return err
}
if err := step(launch, "--out-extension:.js=.mjs"); err != nil {
return err
}
// Plain `.js` output is an ES module; said once.
if err := os.WriteFile(filepath.Join(dir, final, "package.json"), []byte(`{"type":"module","private":true}`+"\n"), 0o644); err != nil {
return err
}
for _, l := range launchers {
path := filepath.Join(dir, final, filepath.FromSlash(l))
if _, err := os.Stat(path); err == nil {
if err := os.Chmod(path, 0o755); err != nil {
return err
}
}
}
return nil
}
// writeLaunchers writes, beside every entrypoint, an executable that imports the entrypoint and serves what it
// registered over MCP on stdio (builder.go:1926-1956; novox/hq ADR 0193). Its answer is each entrypoint's launcher,
// relative to the bundle's root.
func writeLaunchers(root string, a artifact) (map[string]string, error) {
out := map[string]string{}
for _, entry := range a.Entrypoints {
if !strings.HasSuffix(entry, ".js") {
continue
}
launcher := strings.TrimSuffix(entry, ".js") + launcherSuffix
body := "#!/usr/bin/env node\n" +
"// Written by the mesh's builder (novox/hq ADR 0193): serve what " + entry + " registers,\n" +
"// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.\n" +
"import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\n" +
"await import(\"./" + filepath.Base(entry) + "\");\n" +
"await serveRegisteredOverStdio();\n"
path := filepath.Join(root, filepath.FromSlash(launcher))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, err
}
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
return nil, err
}
if err := os.Chmod(path, 0o755); err != nil {
return nil, err
}
out[entry] = launcher
}
return out, nil
}
// sourcesFor turns compiled entrypoints back into what to compile (builder.go:1735-1747).
func sourcesFor(entrypoints []string, out, ext string) []string {
sources := make([]string, 0, len(entrypoints))
for _, e := range entrypoints {
at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/")
sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+ext)
}
return sources
}
func sortedValues(m map[string]string) []string {
out := make([]string, 0, len(m))
for _, v := range m {
out = append(out, v)
}
sort.Strings(out)
return out
}
// readArtifact reads the named artifact of the module.json in dir, which must be a TypeScript bundle with entrypoints.
func readArtifact(dir, name string) (artifact, error) {
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
if err != nil {
return artifact{}, fmt.Errorf("the module has no module.json: %w", err)
}
var m struct {
Build struct {
Artifacts []artifact `json:"artifacts"`
} `json:"build"`
}
if err := json.Unmarshal(raw, &m); err != nil {
return artifact{}, fmt.Errorf("module.json is not JSON: %w", err)
}
for _, a := range m.Build.Artifacts {
if a.Name != name {
continue
}
if a.Language != "typescript" {
return artifact{}, fmt.Errorf("artifact %q is %q, and this module builds typescript", name, a.Language)
}
if len(a.Entrypoints) == 0 {
return artifact{}, fmt.Errorf("artifact %q names no entrypoints", name)
}
return a, nil
}
return artifact{}, fmt.Errorf("module.json declares no artifact %q", name)
}
modules/mesh-build-node/cmd/mesh-build-node/build_integration_test.go
+91
View File
@@ -0,0 +1,91 @@
//go:build integration
package main
import (
"context"
"os"
"os/exec"
"path/filepath"
"slices"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// realMachine installs the real toolchain from the module's own lock (npm ci --ignore-scripts, from the mesh's
// registry for @novox, which reads without a credential) and makes two workspaces holding one fixture bundle.
func realMachine(t *testing.T) (*Module, string) {
t.Helper()
for _, p := range []string{"node", "npm"} {
if _, err := exec.LookPath(p); err != nil {
t.Skipf("NOT RUN: no %s installed where this check runs (the mesh's Go toolchain image holds none); proven live on a build node", p)
}
}
home, cache, work := t.TempDir(), t.TempDir(), t.TempDir()
for _, n := range []string{"package.json", "package-lock.json"} {
raw, err := os.ReadFile(filepath.Join(moduleDir, "toolchain", n))
if err != nil {
t.Fatal(err)
}
write(t, filepath.Join(home, "toolchain", n), string(raw), 0o644)
}
lock, err := ReadRecipe(filepath.Join(home, "toolchain", "package-lock.json"))
if err != nil {
t.Fatal(err)
}
run := forgekey.ExecRunnerFor(0)
m := &Module{Node: "ace", Recipe: lock, Work: work, Run: run, Ctx: context.Background(),
Toolchain: &Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: run, Registry: func() (Registry, error) {
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
}}}
if err := m.Toolchain.Ensure(context.Background(), m.Recipe.Version()); err != nil {
t.Fatal(err)
}
for _, id := range []string{"w1", "w2"} {
for p, body := range map[string]string{
"modules/m/module.json": `{"module":"m","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`,
"modules/m/package.json": `{"name":"m","type":"module","scripts":{"test":"node -e \"process.exit(0)\""}}`,
"modules/m/tools/index.ts": "import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\nexport const f: () => Promise<void> = serveRegisteredOverStdio as () => Promise<void>;\n",
} {
write(t, filepath.Join(work, id, "tree", filepath.FromSlash(p)), body, 0o644)
}
}
return m, work
}
// 1490/AC2: a fixture TypeScript bundle importing @novox/mesh-sdk, built twice by the real tsc and esbuild, has equal
// digests, and its archive holds the bundled .js, the .serve.mjs launcher and package.json.
func TestAFixtureBundleBuiltTwiceHasOneDigest(t *testing.T) {
m, work := realMachine(t)
var digests []string
for _, id := range []string{"w1", "w2"} {
out, err := m.build(context.Background(), map[string]any{"workspace": id, "module": "modules/m", "artifact": "tools"})
if err != nil {
t.Fatal(err)
}
r := out.(BuildResult)
digests = append(digests, r.Digest)
if want := []string{"package.json", "tools/index.js", "tools/index.serve.mjs"}; !slices.Equal(r.Files, want) {
t.Errorf("files = %v, want %v", r.Files, want)
}
}
if digests[0] != digests[1] {
t.Errorf("digests differ: %v", digests)
}
if links := symlinksUnder(t, work); len(links) != 0 {
t.Errorf("symlinks in the workspaces: %v", links)
}
}
// 1491: the real type-check and test script of the same fixture pass the unit level.
func TestTheFixturePassesTheUnitLevel(t *testing.T) {
m, _ := realMachine(t)
out, err := m.test(context.Background(), map[string]any{"workspace": "w1", "module": "modules/m", "level": "unit"})
if err != nil {
t.Fatal(err)
}
if r := out.(*TestResult); !r.Passed || r.NoTests || len(r.Ran) != 2 {
t.Errorf("result = %+v", r)
}
}
modules/mesh-build-node/cmd/mesh-build-node/build_test.go
+256
View File
@@ -0,0 +1,256 @@
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"io"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const bundleManifest = `{"module":"m","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript",` +
`"entrypoints":["tools/index.js"],"loads":["tools/index.js"]%s}]}}`
// esbuildEffect stands in for esbuild's output: for each entry `.mesh-build/<a>/<path>.js` it writes the bundled file
// under --outdir, named as esbuild names it (the input without its extension, then .js or .mjs when --out-extension:.js=.mjs is given).
func esbuildEffect(t *testing.T) func(c call) {
return func(c call) {
if filepath.Base(c.name) != "esbuild" {
return
}
var outdir, outbase, ext string
var entries []string
for _, a := range c.args {
switch {
case strings.HasPrefix(a, "--outdir="):
outdir = strings.TrimPrefix(a, "--outdir=")
case strings.HasPrefix(a, "--outbase="):
outbase = strings.TrimPrefix(a, "--outbase=")
case a == "--out-extension:.js=.mjs":
ext = ".mjs"
case !strings.HasPrefix(a, "--"):
entries = append(entries, a)
}
}
for _, e := range entries {
rel := strings.TrimPrefix(strings.TrimPrefix(e, outbase), "/")
// esbuild names an output after its input without the extension, and adds .js or the one asked for.
rel = strings.TrimSuffix(rel, filepath.Ext(rel))
if ext != "" {
rel += ext
} else {
rel += ".js"
}
write(t, filepath.Join(c.dir, outdir, rel), "bundled "+rel, 0o644)
}
}
}
func buildArgs() map[string]any {
return map[string]any{"workspace": "w1", "module": "modules/m", "artifact": "tools"}
}
func builtMachine(t *testing.T, extra string) (*Module, *fakeMachine, string) {
m, f, work := newMachine(t)
workspaceWith(t, work, map[string]string{"modules/m/module.json": strings.Replace(bundleManifest, "%s", extra, 1),
"modules/m/tools/index.ts": "export {}"})
f.effect = esbuildEffect(t)
return m, f, work
}
// 1490/AC1: tsc and esbuild run with exactly the build agent's flags. The fixture is copied from mesh-controller
// origin/main: internal/builder/toolchain.go:143-147 (tsc: --module NodeNext --moduleResolution NodeNext --target
// ES2022 --rootDir .), builder.go:1637-1705 (--outDir .mesh-build/<artifact> and the sources) and :1971-2039 (esbuild:
// --bundle --platform=node --format=esm --target=node22 --outbase --outdir --log-level=warning, the createRequire
// banner, --out-extension:.js=.mjs for the launchers, --external:x). Where the build agent ran them in the toolchain
// image they run here with the toolchain's real paths, from the module's directory (env -C).
func TestTscAndEsbuildRunWithTheBuildAgentsFlags(t *testing.T) {
m, f, work := builtMachine(t, `,"external":["pg"]`)
if _, err := m.build(context.Background(), buildArgs()); err != nil {
t.Fatal(err)
}
dir := filepath.Join(work, "w1", "stage-tools", "modules", "module")
nm := filepath.Join(m.toolchainDir(), "node_modules")
banner := "--banner:js=import { createRequire as __meshRequire } from 'node:module'; const require = __meshRequire(import.meta.url);"
common := []string{"--bundle", "--platform=node", "--format=esm", "--target=node22",
"--outbase=.mesh-build/tools", "--outdir=.mesh-build/tools.bundled", "--log-level=warning", banner, "--external:pg"}
want := []string{
"node " + nm + "/typescript/bin/tsc --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . --outDir .mesh-build/tools tools/index.ts",
nm + "/esbuild/bin/esbuild .mesh-build/tools/tools/index.js " + strings.Join(common, " "),
nm + "/esbuild/bin/esbuild .mesh-build/tools/tools/index.serve.mjs " + strings.Join(common, " ") + " --out-extension:.js=.mjs",
}
if len(f.calls) != len(want) {
t.Fatalf("%d commands ran, want %d: %v", len(f.calls), len(want), f.calls)
}
for i, c := range f.calls {
if c.dir != dir || joined(c) != want[i] {
t.Errorf("command %d: in %s ran\n%s\nwant in %s\n%s", i, c.dir, joined(c), dir, want[i])
}
}
}
func readArchive(t *testing.T, path string) map[string]os.FileMode {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
zr, err := gzip.NewReader(bytes.NewReader(raw))
if err != nil {
t.Fatal(err)
}
tr := tar.NewReader(zr)
out := map[string]os.FileMode{}
for {
h, err := tr.Next()
if err == io.EOF {
return out
}
if err != nil {
t.Fatal(err)
}
out[h.Name] = os.FileMode(h.Mode)
}
}
// 1490: the step packs the bundle: the archive holds the bundled .js, the .serve.mjs launcher (executable) and the
// package.json that makes it an ES module, and the result is the archive's digest, the same for the same build.
func TestTheBundleIsPackedWithItsLauncherAndPackageJson(t *testing.T) {
m, _, _ := builtMachine(t, ``)
out, err := m.build(context.Background(), buildArgs())
if err != nil {
t.Fatal(err)
}
r := out.(BuildResult)
if !strings.HasPrefix(r.Digest, "sha256:") || r.Bytes == 0 {
t.Errorf("result = %+v", r)
}
if want := []string{"package.json", "tools/index.js", "tools/index.serve.mjs"}; !slices.Equal(r.Files, want) {
t.Errorf("files = %v, want %v", r.Files, want)
}
modes := readArchive(t, r.Archive)
if len(modes) != 3 || modes["tools/index.serve.mjs"]&0o111 == 0 {
t.Errorf("archive = %v, want the three files with the launcher executable", modes)
}
again, err := m.build(context.Background(), buildArgs())
if err != nil || again.(BuildResult).Digest != r.Digest {
t.Errorf("a second build: %v, digest %v, want %s", err, again, r.Digest)
}
launcher, _ := os.ReadFile(filepath.Join(filepath.Dir(r.Archive), "stage-tools", "modules", "module", ".mesh-build", "tools", "tools", "index.serve.mjs"))
if !strings.Contains(string(launcher), `import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";`) || !strings.Contains(string(launcher), `await import("./index.js");`) {
t.Errorf("launcher = %q", launcher)
}
}
// 1490/AC3: no symlink is made anywhere in the workspace, with externals (whose packages are copied) or without.
func TestNoSymlinkIsMadeInTheWorkspace(t *testing.T) {
for _, extra := range []string{``, `,"external":["nats"]`} {
m, _, work := builtMachine(t, extra)
if _, err := m.build(context.Background(), buildArgs()); err != nil {
t.Fatal(err)
}
if links := symlinksUnder(t, filepath.Join(work, "w1")); len(links) != 0 {
t.Errorf("symlinks in the workspace: %v", links)
}
}
}
// 1490: the bundle's SDK comes from the staging directory's node_modules, two levels above the module, and the packed bundle holds
// the toolchain's packages only when the artifact keeps one external, never typescript or esbuild.
func TestThePackagesAreCopiedBesideTheTreeAndIntoTheBundleOnlyForExternals(t *testing.T) {
m, _, work := builtMachine(t, ``)
out, err := m.build(context.Background(), buildArgs())
if err != nil {
t.Fatal(err)
}
nm := filepath.Join(work, "w1", "stage-tools", "node_modules")
for _, p := range []string{"@novox/mesh-sdk/dist/stdio.js", "@types/node/index.d.ts", "nats/lib/nats.js"} {
if _, err := os.Stat(filepath.Join(nm, p)); err != nil {
t.Errorf("the workspace's node_modules lacks %s", p)
}
}
for _, p := range []string{"typescript", "esbuild", "@esbuild"} {
if _, err := os.Stat(filepath.Join(nm, p)); err == nil {
t.Errorf("the toolchain's own %s was copied", p)
}
}
for f := range readArchive(t, out.(BuildResult).Archive) {
if strings.HasPrefix(f, "node_modules") {
t.Errorf("a bundle without externals carries %s", f)
}
}
m, _, _ = builtMachine(t, `,"external":["nats"]`)
out, _ = m.build(context.Background(), buildArgs())
files := out.(BuildResult).Files
if !slices.Contains(files, "node_modules/nats/lib/nats.js") || !slices.Contains(files, "node_modules/@novox/mesh-sdk/dist/stdio.js") {
t.Errorf("files = %v, want the runtime packages beside the bundle", files)
}
for _, f := range files {
if strings.Contains(f, "typescript") || strings.Contains(f, "esbuild") {
t.Errorf("the bundle carries %s", f)
}
}
}
func TestAFailedCompilerFailsTheStepNamingIt(t *testing.T) {
m, f, _ := builtMachine(t, ``)
f.answer["tsc"] = forgekey.Ran{Status: 2, Stdout: tscTypeError}
if _, err := m.build(context.Background(), buildArgs()); err == nil || !strings.Contains(err.Error(), "tsc for tools") || !strings.Contains(err.Error(), "tools/bad.ts") {
t.Errorf("err = %v", err)
}
}
func TestOnlyATypeScriptBundleOfTheModuleIsBuilt(t *testing.T) {
m, _, _ := builtMachine(t, ``)
for _, a := range []string{"nope", ""} {
args := buildArgs()
args["artifact"] = a
if _, err := m.build(context.Background(), args); err == nil {
t.Errorf("artifact %q was built", a)
}
}
}
// 1490 (review 7): the build agent runs a module mounted at /app/modules/module with the toolchain's packages at
// /app/node_modules (mesh-controller builder.go `within`), so esbuild writes the path comments of what it inlines
// relative to the module's directory as `../../node_modules/...`. The step builds at that depth: in a staging
// directory whose module directory has the packages two levels above it, whatever the module's place in the
// repository, so the bundle's bytes (and its digest) equal the build agent's.
func TestTheBuildRunsWithThePackagesTwoLevelsAboveTheModule(t *testing.T) {
for _, module := range []string{"modules/m", "."} {
m, f, work := newMachine(t)
files := map[string]string{"module.json": strings.Replace(bundleManifest, "%s", "", 1), "tools/index.ts": "export {}"}
if module != "." {
files = map[string]string{"modules/m/module.json": strings.Replace(bundleManifest, "%s", "", 1), "modules/m/tools/index.ts": "export {}"}
}
workspaceWith(t, work, files)
f.effect = esbuildEffect(t)
args := buildArgs()
args["module"] = module
if _, err := m.build(context.Background(), args); err != nil {
t.Fatalf("%s: %v", module, err)
}
for _, c := range f.calls {
rel, err := filepath.Rel(c.dir, filepath.Join(c.dir, "..", "..", "node_modules"))
if err != nil || rel != "../../node_modules" {
t.Fatalf("%s: relative path %q (%v)", module, rel, err)
}
if _, err := os.Stat(filepath.Join(c.dir, "..", "..", "node_modules", "@novox", "mesh-sdk", "dist", "stdio.js")); err != nil {
t.Errorf("%s: %s ran in %s, which has no packages at ../../node_modules: %v", module, filepath.Base(c.name), c.dir, err)
}
if filepath.Base(c.dir) != "module" || filepath.Base(filepath.Dir(c.dir)) != "modules" {
t.Errorf("%s: ran in %s, want .../modules/module", module, c.dir)
}
}
if !strings.HasPrefix(f.calls[0].dir, filepath.Join(work, "w1")) {
t.Errorf("%s: staged outside the workspace: %s", module, f.calls[0].dir)
}
}
}
modules/mesh-build-node/cmd/mesh-build-node/check.go
+263
View File
@@ -0,0 +1,263 @@
package main
// The test step (novox/hq ADR 0445 §4; issues 1491 and 1292): the two levels of a TypeScript package's tests, run
// against a fetched workspace with the toolchain's tsc. The unit level is the package's type-check and its
// package.json `test` script; the integration level is its `test:integration` script, with the type-check as its
// setup. Each is timed and judged against its bound; a level over its bound says so in the result and does not fail
// (the measure of the catalogue's TypeScript is a later release's, R4f). A package with neither a script nor .ts files
// has no tests, and the result says so.
//
// **The bounds are the Go module's** (5 s for the unit level; 30 s of setup and 60 s of running for the integration
// level), because ADR 0445 names no figures for TypeScript. They are constants here, said in every result.
import (
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// The bounds of a level (ADR 0445 §4).
const (
UnitBound = 5 * time.Second
IntegrationSetupBound = 30 * time.Second
IntegrationRunBound = 60 * time.Second
)
// TestResult is what a test step passes with, with the field names of mesh-build-go's (issue 1290). The step ends
// passed whenever the level ran, whatever its verdict, and Passed is the verdict; it ends failed only when the tests
// could not be run (no toolchain, a command that cannot start, a workspace that was never fetched).
type TestResult struct {
Workspace string `json:"workspace"`
Module string `json:"module"`
Level string `json:"level"`
Passed bool `json:"passed"`
// Why says, in words, why the level failed or went over its bound, or that it has no tests.
Why string `json:"why,omitempty"`
NoTests bool `json:"no_tests,omitempty"`
// Ran names what ran: "type-check", "test" or "test:integration".
Ran []string `json:"ran,omitempty"`
// Elapsed is the level's running time in seconds (at the unit level the type-check and the test script together);
// Setup is the integration level's setup, the type-check, in seconds. Bound and SetupBound are the bounds the
// level is judged by.
Elapsed float64 `json:"elapsed"`
Setup float64 `json:"setup,omitempty"`
Bound float64 `json:"bound"`
SetupBound float64 `json:"setup_bound,omitempty"`
// OverBound says a bound was exceeded. Unlike mesh-build-go's, an over-bound level still passes: the measure of
// the catalogue's TypeScript against the bounds is a later release's (R4f), not this one's gate.
OverBound bool `json:"over_bound"`
Failed []Failure `json:"failed,omitempty"`
}
// Failure is a check or script that failed; Output is the first lines it wrote.
type Failure struct {
Package string `json:"package"`
Test string `json:"test,omitempty"`
Output string `json:"output,omitempty"`
}
func (m *Module) testTool() StepTool {
return StepTool{
Name: "build_node_test",
Description: "Run a TypeScript package's tests in a fetched workspace: the unit level is the type-check (the toolchain's " +
"tsc) and the package's test script, the integration level its test:integration script. Each level is timed " +
"and judged against its bound (unit 5 s; integration 30 s setup and 60 s running); a level over its bound " +
"says so and does not fail. A package with neither script nor .ts files has no tests. Answers a step id at " +
"once; the step ends on its key in the steps state.",
Input: map[string]any{
"type": "object",
"properties": map[string]any{
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch step answered"},
"module": map[string]any{"type": "string", "description": "the package's directory in the workspace, '.' when it is the root"},
"level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}},
},
"required": []string{"workspace", "level"},
},
Long: true,
Run: m.test,
}
}
func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
workspace, _ := args["workspace"].(string)
module, _ := args["module"].(string)
level, _ := args["level"].(string)
if level != "unit" && level != "integration" {
return nil, fmt.Errorf("level %q is neither unit nor integration", level)
}
root, dir, err := m.moduleDirOf(workspace, module)
if err != nil {
return nil, err
}
scripts, err := packageScripts(dir)
if err != nil {
return nil, err
}
sources, err := typeScriptFiles(dir)
if err != nil {
return nil, err
}
res := &TestResult{Workspace: workspace, Module: module, Level: level}
script := "test"
if level == "integration" {
script = "test:integration"
}
hasScript := scripts[script] != ""
if level == "integration" && !hasScript {
res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test:integration script"
return res, nil
}
if !hasScript && len(sources) == 0 {
res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test script and no .ts files"
return res, nil
}
if err := m.provide(root); err != nil {
return nil, fmt.Errorf("providing the toolchain's packages to the workspace: %w", err)
}
where := module
if where == "" {
where = "."
}
var check, run time.Duration
var failed []Failure
// a command that could not start is not a verdict: the step fails. One that ran and said no is.
ran := func(name string, r forgekey.Ran) error {
if r.Err != "" {
return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr))
}
if r.Status != 0 {
failed = append(failed, Failure{Package: where, Test: name, Output: lines(r.Stdout + "\n" + r.Stderr)})
}
return nil
}
if len(sources) > 0 {
start := m.now()
r := m.runIn(ctx, dir, nil, "node", m.typeCheck(dir, sources)...)
check = m.now().Sub(start)
if err := ran("type-check", r); err != nil {
return nil, err
}
res.Ran = append(res.Ran, "type-check")
}
if hasScript && len(failed) == 0 {
start := m.now()
env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm")}
r := m.runIn(ctx, dir, env, "npm", scriptArgs(script)...)
run = m.now().Sub(start)
if err := ran(script, r); err != nil {
return nil, err
}
res.Ran = append(res.Ran, script)
}
res.Failed = failed
var why []string
for _, f := range failed {
why = append(why, fmt.Sprintf("%s failed in %s: %s", f.Test, f.Package, firstLine(f.Output)))
}
if level == "unit" {
res.Elapsed, res.Bound = (check + run).Seconds(), UnitBound.Seconds()
res.OverBound = check+run > UnitBound
if res.OverBound {
why = append(why, fmt.Sprintf("the unit level took %.2f s, over its bound of %s", res.Elapsed, UnitBound))
}
} else {
res.Setup, res.Elapsed = check.Seconds(), run.Seconds()
res.SetupBound, res.Bound = IntegrationSetupBound.Seconds(), IntegrationRunBound.Seconds()
res.OverBound = check > IntegrationSetupBound || run > IntegrationRunBound
if res.OverBound {
why = append(why, fmt.Sprintf("the integration level took %.2f s of setup and %.2f s of running, over its bounds of %s and %s",
res.Setup, res.Elapsed, IntegrationSetupBound, IntegrationRunBound))
}
}
res.Passed = len(failed) == 0
res.Why = strings.Join(why, "; ")
return res, nil
}
// lines is the first lines of a command's output.
func lines(out string) string {
var keep []string
for _, l := range strings.Split(out, "\n") {
if l = strings.TrimSpace(l); l != "" {
keep = append(keep, l)
if len(keep) == 15 {
break
}
}
}
return strings.Join(keep, "\n")
}
func firstLine(s string) string {
if i := strings.IndexByte(strings.TrimSpace(s), '\n'); i >= 0 {
return strings.TrimSpace(s)[:i]
}
return strings.TrimSpace(s)
}
// scriptArgs is npm's line for a script: `npm test` for test, `npm run <name>` for another.
func scriptArgs(script string) []string {
if script == "test" {
return []string{"test"}
}
return []string{"run", script}
}
// typeCheck is tsc's line for the type-check: the package's tsconfig.json when it has one, else the build agent's
// flags (toolchain.go:143-147) with --noEmit over every .ts file of the package.
func (m *Module) typeCheck(dir string, sources []string) []string {
if _, err := os.Stat(filepath.Join(dir, "tsconfig.json")); err == nil {
return []string{m.tsc(), "--noEmit", "-p", "."}
}
return append([]string{m.tsc(), "--noEmit", "--module", "NodeNext", "--moduleResolution", "NodeNext",
"--target", "ES2022", "--rootDir", "."}, sources...)
}
// packageScripts are the scripts of the package.json in dir (none without one).
func packageScripts(dir string) (map[string]string, error) {
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
if os.IsNotExist(err) {
return map[string]string{}, nil
}
if err != nil {
return nil, err
}
var p struct {
Scripts map[string]string `json:"scripts"`
}
if err := json.Unmarshal(raw, &p); err != nil {
return nil, fmt.Errorf("the package.json is not JSON: %w", err)
}
return p.Scripts, nil
}
// typeScriptFiles are the .ts files of the package (declaration files excepted), relative and sorted, without
// node_modules, hidden directories and a build's output.
func typeScriptFiles(dir string) ([]string, error) {
var out []string
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
name := d.Name()
if d.IsDir() {
if path != dir && (name == "node_modules" || strings.HasPrefix(name, ".")) {
return filepath.SkipDir
}
return nil
}
if strings.HasSuffix(name, ".ts") && !strings.HasSuffix(name, ".d.ts") {
rel, _ := filepath.Rel(dir, path)
out = append(out, filepath.ToSlash(rel))
}
return nil
})
return out, err
}
modules/mesh-build-node/cmd/mesh-build-node/check_test.go
+199
View File
@@ -0,0 +1,199 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// tscTypeError is what tsc 5.9.3 printed for `export const n: number = "text";` in tools/bad.ts, run on this machine
// on 2026-10-20 with the flags of the type-check below (status 2).
const tscTypeError = "tools/bad.ts(1,14): error TS2322: Type 'string' is not assignable to type 'number'.\n"
func testArgs(level string) map[string]any {
return map[string]any{"workspace": "w1", "module": ".", "level": level}
}
// 1491/AC1: a package with a type error fails the unit level, naming the file.
func TestATypeErrorFailsTheUnitLevelNamingTheFile(t *testing.T) {
m, f, work := newMachine(t)
workspaceWith(t, work, map[string]string{"tools/bad.ts": `export const n: number = "text";`, "package.json": `{"name":"x"}`})
f.answer["tsc"] = forgekey.Ran{Status: 2, Stdout: tscTypeError}
out, err := m.test(context.Background(), testArgs("unit"))
if err != nil {
t.Fatalf("a type error ended the step %v, want it finished with the verdict in its result", err)
}
r := out.(*TestResult)
if r.Passed || len(r.Failed) != 1 || !strings.Contains(r.Why, "tools/bad.ts") || !strings.Contains(r.Failed[0].Output, "TS2322") {
t.Fatalf("result = %+v, want passed false, naming tools/bad.ts", r)
}
if c := f.calls[0]; !strings.Contains(joined(c), "--noEmit") || !strings.Contains(joined(c), "typescript/bin/tsc") ||
!strings.Contains(joined(c), "--moduleResolution NodeNext --target ES2022") || !strings.HasSuffix(joined(c), "tools/bad.ts") {
t.Errorf("the type-check ran %s", joined(c))
}
}
// 1491/AC2: a package with a passing test script passes the unit level, which reports its time; over its bound it
// says so and still passes.
func TestAPassingTestScriptPassesTheUnitLevelAndReportsItsTime(t *testing.T) {
m, f, work := newMachine(t)
workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "package.json": `{"scripts":{"test":"node --test"}}`})
f.cost["tsc"], f.cost["npm"] = 400*time.Millisecond, 600*time.Millisecond
out, err := m.test(context.Background(), testArgs("unit"))
if err != nil {
t.Fatal(err)
}
r := out.(*TestResult)
if !r.Passed || r.NoTests || r.Elapsed != 1 || r.OverBound || r.Bound != 5 || strings.Join(r.Ran, ",") != "type-check,test" {
t.Errorf("result = %+v, want the type-check and the test script in 1 s within the bound", r)
}
last := f.calls[len(f.calls)-1]
if last.name != "npm" || strings.Join(last.args, " ") != "test" {
t.Errorf("ran %s, want npm test", joined(last))
}
f.cost["npm"] = 6 * time.Second
out, err = m.test(context.Background(), testArgs("unit"))
if r := out.(*TestResult); err != nil || !r.OverBound {
t.Errorf("a slow unit level = %+v, %v, want passed and over_bound", r, err)
}
}
// 1491/AC3: a package with neither script nor .ts files has no tests, and nothing runs.
func TestAPackageWithNeitherScriptNorTypeScriptHasNoTests(t *testing.T) {
m, f, work := newMachine(t)
workspaceWith(t, work, map[string]string{"package.json": `{"name":"x"}`, "node_modules/p/x.ts": "ignored", "types.d.ts": "declare const a: 1;"})
for _, level := range []string{"unit", "integration"} {
out, err := m.test(context.Background(), testArgs(level))
r := out.(*TestResult)
if err != nil || !r.Passed || !r.NoTests || !strings.Contains(r.Why, "no tests") {
t.Errorf("%s = %+v, %v, want the result to say no tests", level, r, err)
}
}
if len(f.calls) != 0 {
t.Errorf("%d commands ran for a package with no tests", len(f.calls))
}
}
// 1491/AC4: with test:integration present the integration level runs it, and reports the setup (the type-check) and
// the running time apart.
func TestTheIntegrationLevelRunsItsScriptAndReportsSetupAndRunningApart(t *testing.T) {
m, f, work := newMachine(t)
workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "tsconfig.json": `{}`,
"package.json": `{"scripts":{"test":"x","test:integration":"node --test int"}}`})
f.cost["tsc"], f.cost["npm"] = 2*time.Second, 40*time.Second
out, err := m.test(context.Background(), testArgs("integration"))
if err != nil {
t.Fatal(err)
}
r := out.(*TestResult)
if r.Setup != 2 || r.Elapsed != 40 || r.SetupBound != 30 || r.Bound != 60 || !r.Passed || r.OverBound || r.Level != "integration" {
t.Errorf("result = %+v, want setup 2 s and running 40 s apart", r)
}
last := f.calls[len(f.calls)-1]
if strings.Join(last.args, " ") != "run test:integration" {
t.Errorf("ran %s, want npm run test:integration", joined(last))
}
if tsc := f.calls[0]; strings.Join(tsc.args[1:], " ") != "--noEmit -p ." {
t.Errorf("a package with a tsconfig was checked with %s", joined(tsc))
}
f.cost["npm"] = 61 * time.Second
out, _ = m.test(context.Background(), testArgs("integration"))
if !out.(*TestResult).OverBound {
t.Errorf("61 s of running is within the bound")
}
}
// The commands of a step get an environment built from scratch: no word of this process's.
func TestTheStepsCommandsGetAnEnvironmentOfTheirOwn(t *testing.T) {
t.Setenv("MESH_SECRET_TOKEN", "hunter2")
m, f, work := newMachine(t)
workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "package.json": `{"scripts":{"test":"x"}}`})
if _, err := m.test(context.Background(), testArgs("unit")); err != nil {
t.Fatal(err)
}
for _, c := range f.calls {
for _, e := range c.env {
if strings.Contains(e, "hunter2") || strings.HasPrefix(e, "MESH_") {
t.Errorf("%s was given %s", joined(c), e)
}
}
if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") {
t.Errorf("%s env = %v, want PATH, HOME and the locale", c.name, c.env)
}
}
}
// 1491: the step finishes with the verdict when the tests fail, and fails only when they could not be run: a failing
// test script is a verdict, a program that cannot start is not.
func TestAFailingScriptIsAVerdictAndACommandThatCannotStartIsAFailedStep(t *testing.T) {
m, f, work := newMachine(t)
workspaceWith(t, work, map[string]string{"package.json": `{"scripts":{"test":"x"}}`})
f.answer["npm"] = forgekey.Ran{Status: 1, Stdout: "not ok 1 - adds\n"}
out, err := m.test(context.Background(), testArgs("unit"))
r, _ := out.(*TestResult)
if err != nil || r == nil || r.Passed || len(r.Failed) != 1 || r.Failed[0].Test != "test" || !strings.Contains(r.Failed[0].Output, "not ok 1") {
t.Errorf("a failing script = %+v, %v, want passed false and its output", r, err)
}
f.answer["npm"] = forgekey.Ran{Status: 127, Err: "ENOENT"}
if out, err := m.test(context.Background(), testArgs("unit")); err == nil {
t.Errorf("npm not installed = %v, want the step to fail", out)
}
if _, err := m.test(context.Background(), map[string]any{"workspace": "nope", "level": "unit"}); err == nil {
t.Errorf("a workspace never fetched did not fail the step")
}
}
func TestAWorkspaceOutsideTheWorkDirectoryIsRefused(t *testing.T) {
m, _, _ := newMachine(t)
for _, a := range []map[string]any{
{"workspace": "../etc", "level": "unit"},
{"workspace": "w1", "module": "../x", "level": "unit"},
{"workspace": "w1", "module": "/etc", "level": "unit"},
{"workspace": "w1", "level": "smoke"},
} {
if _, err := m.test(context.Background(), a); err == nil {
t.Errorf("%v was accepted", a)
}
}
}
// 1491 (review 6): --no-bin-links leaves the toolchain no bin directory, so `npm run build` found no tsc or esbuild
// and exited 127, which read as a failed test. The workspace now has a bin directory of small shell scripts (files,
// not links) that call the toolchain's tsc and esbuild, first on the PATH of the test script.
func TestTheTestScriptFindsTscAndEsbuildOnItsPath(t *testing.T) {
m, f, work := newMachine(t)
workspaceWith(t, work, map[string]string{"package.json": `{"scripts":{"test":"tsc && esbuild"}}`})
if _, err := m.test(context.Background(), testArgs("unit")); err != nil {
t.Fatal(err)
}
bin := filepath.Join(work, "w1", "bin")
nm := filepath.Join(m.toolchainDir(), "node_modules")
for name, want := range map[string]string{
"tsc": "exec node '" + nm + "/typescript/bin/tsc' \"$@\"",
"esbuild": "exec '" + nm + "/esbuild/bin/esbuild' \"$@\"",
} {
info, err := os.Lstat(filepath.Join(bin, name))
if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 {
t.Fatalf("bin/%s: %v %v, want an executable file", name, info, err)
}
raw, _ := os.ReadFile(filepath.Join(bin, name))
if !strings.HasPrefix(string(raw), "#!/bin/sh\n") || !strings.Contains(string(raw), want) {
t.Errorf("bin/%s = %q, want a script calling the toolchain's program", name, raw)
}
}
npm := f.calls[len(f.calls)-1]
path := ""
for _, e := range npm.env {
if strings.HasPrefix(e, "PATH=") {
path = strings.TrimPrefix(e, "PATH=") // the last PATH wins
}
}
if npm.name != "npm" || !strings.HasPrefix(path, bin+":") {
t.Errorf("npm ran with PATH %q, want %s first", path, bin)
}
}
modules/mesh-build-node/cmd/mesh-build-node/emit_test.go
+55
View File
@@ -0,0 +1,55 @@
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"slices"
"testing"
)
// Issues 1483 and 1488 (for the indexer, issue 1444): the guard emits toolchain.reported after each write of the
// toolchains key, with the node and the language.
func TestTheGuardEmitsToolchainReportedAfterEachWrite(t *testing.T) {
m := moduleWith(newStates(), "v26.10.0")
type event struct {
typ string
body map[string]string
}
var got []event
m.Emit = func(typ string, body any) error {
raw, _ := json.Marshal(body)
var b map[string]string
json.Unmarshal(raw, &b)
got = append(got, event{typ, b})
return nil
}
if err := m.Check(context.Background()); err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].typ != "toolchain.reported" || got[0].body["node"] != "ace" || got[0].body["language"] != "typescript" || len(got[0].body) != 2 {
t.Errorf("events = %+v, want one toolchain.reported for ace, typescript", got)
}
m.Check(context.Background())
if len(got) != 2 {
t.Errorf("a second guard pass emitted %d events in all, want 2", len(got))
}
}
// The event is carried by the module's stream, not named in emits (ADR 0363: one place); the validator checks that.
func TestTheManifestDeclaresTheToolchainsStream(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m struct {
Streams []struct {
Name string `json:"name"`
Events []string `json:"events"`
} `json:"streams"`
}
if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) != 1 || m.Streams[0].Name != "toolchains-node" || !slices.Equal(m.Streams[0].Events, []string{"toolchain.**"}) {
t.Errorf("streams = %+v (%v)", m.Streams, err)
}
}
modules/mesh-build-node/cmd/mesh-build-node/fetch.go
+298
View File
@@ -0,0 +1,298 @@
package main
// The fetch step (novox/hq ADR 0405 §5, ADR 0466 §4 and §5, ADR 0469 §2 and §5; issue 1489): a commit of a repository
// of the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and for each
// TypeScript module in it the packages its package.json names beyond the SDK, installed from its committed lock.
//
// **Only what the module itself depends on is installed.** The SDK and nats come with the toolchain, so a bundle that
// names only @novox/mesh-sdk (and build-time types and typescript) resolves nothing. One that names more must carry a
// package-lock.json, or the step fails naming its directory. The install is the build agent's (mesh-controller
// internal/builder/dependencies.go:49-95): in a scratch copy of package.json and the lock with the SDK, devDependencies
// and scripts taken out, `npm ci --omit=dev --omit=peer --ignore-scripts`, the install's copies of the SDK removed, and
// the result put at the module's node_modules. This module adds --no-bin-links, so the workspace holds no symlink.
import (
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1).
const ForgeAlias = "mesh-forge"
// sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
const sdkPackage = "@novox/mesh-sdk"
// WorkspaceAge is how long a workspace is kept.
const WorkspaceAge = 24 * time.Hour
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
// FetchResult is what a passed fetch step says.
type FetchResult struct {
Workspace string `json:"workspace"`
Commit string `json:"commit"`
// Modules are the TypeScript modules' directories found, and Installed those whose own packages were installed.
Modules []string `json:"modules"`
Installed []string `json:"installed"`
// Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK
// with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459).
Refused []Refusal `json:"refused"`
}
// Refusal is a module a fetch did not install the packages of.
type Refusal struct {
Module string `json:"module"`
Reason string `json:"reason"`
}
func (m *Module) fetchTool() StepTool {
return StepTool{
Name: "build_node_fetch",
Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
"workspace of its own, and install each TypeScript module's own packages from its lock (a module naming only the " +
"SDK installs nothing). Answers a step id at once; the step ends on its key in the steps state with the workspace's id.",
Input: map[string]any{
"type": "object",
"properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
},
"required": []string{"repository", "commit"},
},
Long: true,
Run: m.fetch,
}
}
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
repository, _ := args["repository"].(string)
commit, _ := args["commit"].(string)
ref, _ := args["ref"].(string)
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
}
m.CleanWorkspaces(m.now())
id := buildstep.NewID()
root := filepath.Join(m.Work, id)
tree := filepath.Join(root, "tree")
if err := os.MkdirAll(tree, 0o700); err != nil {
return nil, err
}
// git and ssh need PATH and the account's HOME, which a clean runner does not supply.
gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
}
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil {
return nil, err
}
dirs, err := m.typeScriptModules(ctx, tree)
if err != nil {
return nil, err
}
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
for _, d := range dirs {
res.Modules = append(res.Modules, d)
did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
if err != nil {
return nil, err
}
if refused != "" {
res.Refused = append(res.Refused, Refusal{Module: d, Reason: refused})
}
if did {
res.Installed = append(res.Installed, d)
}
}
return res, nil
}
// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to
// it, in order.
func (m *Module) typeScriptModules(ctx context.Context, tree string) ([]string, error) {
r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
if r.Status != 0 || r.Err != "" {
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
}
var dirs []string
for _, l := range strings.Split(r.Stdout, "\n") {
if l = strings.TrimSpace(l); l == "" {
continue
}
raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l)))
if err != nil {
return nil, err
}
var mj struct {
Build struct {
Artifacts []artifact `json:"artifacts"`
} `json:"build"`
}
if json.Unmarshal(raw, &mj) != nil {
continue // a module.json the build step will refuse by name, not a fetch's business
}
for _, a := range mj.Build.Artifacts {
if a.Language == "typescript" {
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
break
}
}
}
sort.Strings(dirs)
return dirs, nil
}
// ownDependencies are the package.json's dependencies beyond the SDK and beyond what is build-time: the types and
// typescript, which the toolchain carries (dependencies.go:49-70). Sorted; none for a module without a package.json.
func ownDependencies(dir string) (names []string, pkg map[string]any, err error) {
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
if os.IsNotExist(err) {
return nil, nil, nil
}
if err != nil {
return nil, nil, err
}
if err := json.Unmarshal(raw, &pkg); err != nil {
return nil, nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
}
deps, _ := pkg["dependencies"].(map[string]any)
for name := range deps {
if name != sdkPackage && name != "typescript" && !strings.HasPrefix(name, "@types/") {
names = append(names, name)
}
}
sort.Strings(names)
return names, pkg, nil
}
// lockProblem says why a module's own packages cannot be installed: it names packages beyond the SDK and has no
// package-lock.json. Empty when it names none or has its lock.
func lockProblem(dir, rel string) (string, error) {
names, _, err := ownDependencies(dir)
if err != nil || len(names) == 0 {
return "", err
}
if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
return fmt.Sprintf("%s names packages beyond the SDK and has no package-lock.json (%s): commit one", rel, strings.Join(names, ", ")), nil
}
return "", nil
}
// installOwn installs a module's own packages into its directory, or does nothing for a module that has none. A module
// that names packages and has no lock is not installed and not an error: the reason is answered, the fetch passes and
// the build of that bundle fails naming it.
func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (installed bool, refused string, err error) {
names, pkg, err := ownDependencies(dir)
if err != nil || len(names) == 0 {
return false, "", err
}
if why, err := lockProblem(dir, rel); why != "" || err != nil {
return false, why, err
}
lock, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
if err != nil {
return false, "", err
}
if m.Toolchain == nil || m.Toolchain.Registry == nil {
return false, "", fmt.Errorf("the module has no package registry to install %s from", rel)
}
reg, err := m.Toolchain.Registry()
if err != nil {
return false, "", err
}
// A scratch copy, so the module's package.json and lock are never rewritten (dependencies.go installSteps).
scratch := filepath.Join(root, "npm-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
if err := os.MkdirAll(scratch, 0o700); err != nil {
return false, "", err
}
for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies"} {
if d, ok := pkg[k].(map[string]any); ok {
delete(d, sdkPackage)
}
}
delete(pkg, "devDependencies")
delete(pkg, "scripts")
raw, _ := json.Marshal(pkg)
if err := os.WriteFile(filepath.Join(scratch, "package.json"), raw, 0o600); err != nil {
return false, "", err
}
if err := os.WriteFile(filepath.Join(scratch, "package-lock.json"), lock, 0o600); err != nil {
return false, "", err
}
// The registry is given through a config file of its own, never on the command line, and removed after.
npmrc := filepath.Join(scratch, ".npmrc-install")
if err := os.WriteFile(npmrc, []byte(reg.Npmrc()), 0o600); err != nil {
return false, "", err
}
env := append(m.baseEnv(), "npm_config_cache="+filepath.Join(m.Toolchain.Paths.Cache, "npm"))
r := m.Run(ctx, env, "npm", "ci", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links",
"--no-audit", "--no-fund", "--prefix", scratch, "--userconfig", npmrc)
os.Remove(npmrc)
if why := ranFailed(r); why != "" {
return false, "", fmt.Errorf("installing the packages of %s (%s): %s", rel, strings.Join(names, ", "), why)
}
nm := filepath.Join(scratch, "node_modules")
if err := removeSDKCopies(nm); err != nil {
return false, "", err
}
if err := os.RemoveAll(filepath.Join(dir, "node_modules")); err != nil {
return false, "", err
}
if err := os.Rename(nm, filepath.Join(dir, "node_modules")); err != nil {
return false, "", err
}
return true, "", os.RemoveAll(scratch)
}
// removeSDKCopies removes every copy of the SDK the install pulled in: the toolchain's is the pin.
func removeSDKCopies(nm string) error {
var found []string
err := filepath.WalkDir(nm, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() && strings.HasSuffix(filepath.ToSlash(p), "/node_modules/@novox/mesh-sdk") {
found = append(found, p)
return fs.SkipDir
}
return nil
})
if err != nil {
return err
}
for _, p := range found {
if err := os.RemoveAll(p); err != nil {
return err
}
}
return nil
}
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
func (m *Module) CleanWorkspaces(now time.Time) {
entries, err := os.ReadDir(m.Work)
if err != nil {
return
}
for _, e := range entries {
info, err := e.Info()
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
continue
}
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
}
}
}
modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go
+303
View File
@@ -0,0 +1,303 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const commitSHA = "0123456789abcdef0123456789abcdef01234567"
var fetchArgs = map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}
// fetchMachine is a machine whose git checks out the given files into the tree it is asked to fetch into (the fake
// answers the commands of FetchCommit and `git ls-files`), and whose npm ci writes the packages the test names.
func fetchMachine(t *testing.T, files map[string]string, installs map[string]string) (*Module, *fakeMachine) {
t.Helper()
m, f, _ := newMachine(t)
f.answer["git rev-parse"] = forgekey.Ran{Stdout: commitSHA + "\n"}
var manifests []string
for p := range files {
if strings.HasSuffix(p, "module.json") {
manifests = append(manifests, p)
}
}
f.answer["git ls-files"] = forgekey.Ran{Stdout: strings.Join(manifests, "\n") + "\n"}
f.effect = func(c call) {
switch {
case c.name == "git" && slices_contains(c.args, "checkout"):
for i, a := range c.args {
if a == "-C" {
for p, body := range files {
write(t, filepath.Join(c.args[i+1], filepath.FromSlash(p)), body, 0o644)
}
}
}
case c.name == "npm":
for i, a := range c.args {
if a == "--prefix" {
for p, body := range installs {
write(t, filepath.Join(c.args[i+1], "node_modules", filepath.FromSlash(p)), body, 0o644)
}
}
}
}
}
return m, f
}
func slices_contains(l []string, s string) bool {
for _, x := range l {
if x == s {
return true
}
}
return false
}
func typeScriptModule(pkg string) map[string]string {
return map[string]string{
"modules/m/module.json": `{"module":"m","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`,
"modules/m/package.json": pkg,
}
}
func npmCalls(f *fakeMachine) []call {
var out []call
for _, c := range f.calls {
if c.name == "npm" {
out = append(out, c)
}
}
return out
}
// 1489/AC1: a module whose package.json names only @novox/mesh-sdk and @types/node (and typescript) installs nothing
// and the step passes.
func TestAModuleNamingOnlyTheSdkAndTypesInstallsNothing(t *testing.T) {
m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"},"devDependencies":{"@types/node":"^22","typescript":"^5"}}`), nil)
out, err := m.fetch(context.Background(), fetchArgs)
if err != nil {
t.Fatal(err)
}
r := out.(FetchResult)
if len(npmCalls(f)) != 0 || len(r.Installed) != 0 || len(r.Modules) != 1 || r.Modules[0] != "modules/m" || r.Commit != commitSHA {
t.Errorf("result = %+v, npm calls %d, want the module found and nothing installed", r, len(npmCalls(f)))
}
if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "module.json")); err != nil {
t.Errorf("the commit is not checked out in the workspace: %v", err)
}
}
// 1489/AC2: a module naming lodash with a lock gets npm ci --omit=dev --ignore-scripts (fake runner), its install put
// at its node_modules without the SDK and without any copy of it; without a lock the step fails naming the directory.
func TestAModuleNamingLodashWithALockIsInstalledWithoutTheSdkAndWithoutALockFails(t *testing.T) {
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"},"devDependencies":{"x":"1"},"scripts":{"test":"y"}}`)
files["modules/m/package-lock.json"] = `{"lockfileVersion":3}`
m, f := fetchMachine(t, files, map[string]string{"lodash/index.js": "l", "@novox/mesh-sdk/index.js": "sdk", "a/node_modules/@novox/mesh-sdk/i.js": "nested"})
out, err := m.fetch(context.Background(), fetchArgs)
if err != nil {
t.Fatal(err)
}
r := out.(FetchResult)
calls := npmCalls(f)
if len(calls) != 1 {
t.Fatalf("npm ran %d times, want once", len(calls))
}
line := strings.Join(calls[0].args, " ")
if !strings.HasPrefix(line, "ci --omit=dev --omit=peer --ignore-scripts --no-bin-links") {
t.Errorf("ran npm %s", line)
}
nm := filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules")
if _, err := os.Stat(filepath.Join(nm, "lodash", "index.js")); err != nil {
t.Errorf("lodash is not at the module's node_modules: %v", err)
}
for _, gone := range []string{"@novox/mesh-sdk", "a/node_modules/@novox/mesh-sdk"} {
if _, err := os.Stat(filepath.Join(nm, gone)); err == nil {
t.Errorf("a copy of the SDK is left at %s", gone)
}
}
if got := strings.Join(r.Installed, ","); got != "modules/m" {
t.Errorf("installed = %q", got)
}
// the module's own package.json is untouched: the install worked on a scratch copy.
if raw, _ := os.ReadFile(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "package.json")); !strings.Contains(string(raw), "scripts") {
t.Errorf("the module's package.json was rewritten: %s", raw)
}
var scratch []string
for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) {
scratch = append(scratch, e)
}
if strings.Contains(strings.Join(scratch, ","), "npm-") {
t.Errorf("the scratch install is left in the workspace: %v", scratch)
}
}
// 1489 (the 1459 settlement): a module naming packages with no package-lock.json does not fail the fetch: it passes,
// installs nothing for that module, and its result lists the refused bundle with its reason; a build of that bundle
// fails naming it.
func TestAModuleWithoutALockIsRefusedInTheResultNotTheFetch(t *testing.T) {
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"}}`)
files["modules/ok/module.json"] = `{"module":"ok","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["tools/index.js"]}]}}`
files["modules/ok/package.json"] = `{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`
m, f := fetchMachine(t, files, nil)
out, err := m.fetch(context.Background(), fetchArgs)
if err != nil {
t.Fatalf("a module without a lock failed the fetch: %v", err)
}
r := out.(FetchResult)
if len(npmCalls(f)) != 0 || len(r.Installed) != 0 || len(r.Modules) != 2 {
t.Errorf("result = %+v, npm calls %d", r, len(npmCalls(f)))
}
if len(r.Refused) != 1 || r.Refused[0].Module != "modules/m" ||
!strings.Contains(r.Refused[0].Reason, "names packages beyond the SDK and has no package-lock.json") || !strings.Contains(r.Refused[0].Reason, "lodash") {
t.Fatalf("refused = %+v, want modules/m with its reason", r.Refused)
}
// the build of that bundle fails naming it, without running a compiler.
f.calls = nil
_, err = m.build(context.Background(), map[string]any{"workspace": r.Workspace, "module": "modules/m", "artifact": "tools"})
if err == nil || !strings.Contains(err.Error(), "modules/m") || !strings.Contains(err.Error(), "no package-lock.json") || len(f.calls) != 0 {
t.Errorf("build = %v after %d commands, want the refusal naming modules/m and nothing run", err, len(f.calls))
}
}
// 1489 (review 4): a bundle depending on another @novox package is installed, the mesh's registry given to npm ci
// through --userconfig, as the toolchain's install gives it.
func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *testing.T) {
files := typeScriptModule(`{"dependencies":{"@novox/other":"^1","lodash":"^4"}}`)
files["modules/m/package-lock.json"] = `{}`
m, f := fetchMachine(t, files, map[string]string{"@novox/other/index.js": "o", "lodash/index.js": "l"})
m.Toolchain.Registry = func() (Registry, error) {
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil
}
var npmrc string
inner := f.effect
f.effect = func(c call) {
inner(c)
for i, a := range c.args {
if c.name == "npm" && a == "--userconfig" {
raw, _ := os.ReadFile(c.args[i+1])
npmrc = string(raw)
}
}
}
out, err := m.fetch(context.Background(), fetchArgs)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || !strings.Contains(npmrc, ":_auth=") {
t.Errorf("npm ci was given the config %q", npmrc)
}
if strings.Contains(strings.Join(npmCalls(f)[0].args, " "), "pw") {
t.Errorf("the credential is on npm's command line")
}
r := out.(FetchResult)
if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules", "@novox", "other", "index.js")); err != nil {
t.Errorf("@novox/other was not installed: %v", err)
}
for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) {
if strings.HasPrefix(e, "npm-") {
t.Errorf("the scratch install (and its credential file) is left: %s", e)
}
}
}
func mustReadDir(t *testing.T, dir string) []string {
t.Helper()
es, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
var out []string
for _, e := range es {
out = append(out, e.Name())
}
return out
}
// 1489/AC3: given a process environment holding a token, no command of the step sees it. The commands the step makes
// are given PATH, HOME and the locale and their own words; and the real clean runner, run on `env` here, prints only
// those.
func TestNoCommandOfTheStepSeesATokenOfTheProcess(t *testing.T) {
t.Setenv("MESH_BUS_CREDENTIAL", "hunter2")
t.Setenv("NPM_TOKEN", "hunter3")
files := typeScriptModule(`{"dependencies":{"lodash":"^4"}}`)
files["modules/m/package-lock.json"] = `{}`
m, f := fetchMachine(t, files, map[string]string{"lodash/index.js": "l"})
if _, err := m.fetch(context.Background(), fetchArgs); err != nil {
t.Fatal(err)
}
if len(f.calls) < 6 {
t.Fatalf("only %d commands ran", len(f.calls))
}
for _, c := range f.calls {
for _, e := range c.env {
if strings.Contains(e, "hunter") || strings.HasPrefix(e, "MESH_") || strings.HasPrefix(e, "NPM_TOKEN") {
t.Errorf("%s %v was given %s", c.name, c.args[:min(len(c.args), 3)], e)
}
}
if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") || c.env[2] != "LC_ALL=C" {
t.Errorf("%s was given %v, want PATH, HOME and the locale first", c.name, c.env)
}
}
// The runner main wires: what it hands a command is exactly what the step gave.
ran := forgekey.ExecRunnerClean(10*time.Second)(context.Background(), append(m.baseEnv(), "ONLY=this"), "env")
if ran.Status != 0 || strings.Contains(ran.Stdout, "hunter") || !strings.Contains(ran.Stdout, "ONLY=this") {
t.Errorf("the clean runner printed %q", ran.Stdout)
}
}
func TestFetchRefusesWhatIsNotARepositoryBeforeAnythingRuns(t *testing.T) {
m, f := fetchMachine(t, nil, nil)
for _, repo := range []string{"", "x", "../etc/passwd", "a/b/c", "a/..", "o/r;rm"} {
if _, err := m.fetch(context.Background(), map[string]any{"repository": repo, "commit": commitSHA}); err == nil {
t.Errorf("%q was accepted", repo)
}
}
if len(f.calls) != 0 {
t.Errorf("%d commands ran", len(f.calls))
}
}
func TestFetchAsksGitForTheSshRemote(t *testing.T) {
m, f := fetchMachine(t, typeScriptModule(`{}`), nil)
if _, err := m.fetch(context.Background(), fetchArgs); err != nil {
t.Fatal(err)
}
var all []string
for _, c := range f.calls {
all = append(all, c.name+" "+strings.Join(c.args, " "))
}
lines := strings.Join(all, "\n")
for _, want := range []string{"remote add origin ssh://mesh-forge/novox/mesh-catalog.git", "fetch --no-tags --depth 1 origin " + commitSHA, "checkout -q --detach " + commitSHA} {
if !strings.Contains(lines, want) {
t.Errorf("git was not asked %q:\n%s", want, lines)
}
}
}
func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) {
m, _, work := newMachine(t)
old, fresh := filepath.Join(work, "aaaa"), filepath.Join(work, "bbbb")
for _, d := range []string{old, fresh, filepath.Join(work, "Not-A-Workspace")} {
os.MkdirAll(d, 0o755)
}
os.Chtimes(old, time.Now().Add(-25*time.Hour), time.Now().Add(-25*time.Hour))
os.Chtimes(filepath.Join(work, "Not-A-Workspace"), time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
m.CleanWorkspaces(time.Now())
if _, err := os.Stat(old); err == nil {
t.Errorf("the old workspace is kept")
}
if _, err := os.Stat(fresh); err != nil {
t.Errorf("the fresh workspace was removed")
}
if _, err := os.Stat(filepath.Join(work, "Not-A-Workspace")); err != nil {
t.Errorf("a directory that is no workspace was removed")
}
}
modules/mesh-build-node/cmd/mesh-build-node/forge_test.go
+247
View File
@@ -0,0 +1,247 @@
package main
// The forge side of a build module (novox/hq ADR 0466 §2 and §4, issue 1484). This file is the same for every build
// module: mesh-build-node takes a copy and changes the three constants below.
import (
"context"
"encoding/base64"
"encoding/binary"
"encoding/json"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const (
forgeModule = "mesh-build-node"
forgeHome = "/var/lib/mesh-build-node"
forgeModuleD = "../.." // the module's directory, where module.json and files/ are
)
type forgeManifest struct {
Requires []any `json:"requires"`
State []any `json:"state"`
Resources []map[string]any `json:"resources"`
Build struct {
Artifacts []map[string]any `json:"artifacts"`
} `json:"build"`
}
func readForgeManifest(t *testing.T) forgeManifest {
t.Helper()
raw, err := os.ReadFile(filepath.Join(forgeModuleD, "module.json"))
if err != nil {
t.Fatal(err)
}
var m forgeManifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
return m
}
func forgeTemplate(t *testing.T, r map[string]any) string {
t.Helper()
name, _ := r["template"].(string)
if name == "" {
t.Fatalf("resource %v names no template", r["id"])
}
raw, err := os.ReadFile(filepath.Join(forgeModuleD, filepath.FromSlash(name)))
if err != nil {
t.Fatalf("resource %v names %s: %v", r["id"], name, err)
}
return string(raw)
}
func forgeResource(t *testing.T, m forgeManifest, id string) map[string]any {
t.Helper()
for _, r := range m.Resources {
if r["id"] == id {
if r["owner"] != forgeModule {
t.Errorf("%s is owned by %v, not the module's own account", id, r["owner"])
}
if p, _ := r["path"].(string); !strings.HasPrefix(p, "${dir:") || strings.Contains(p, "account-home") {
t.Errorf("%s is at %q: a path of the account's own home (${dir:...}), never the operator's", id, p)
}
return r
}
}
t.Fatalf("no resource %q", id)
return nil
}
var forgeSample = map[string]string{
"known_host": "[novox.internal]:222 ssh-ed25519 AAAAC3Nza",
"host_key": "ssh-ed25519 AAAAC3Nza",
"ssh_host": "novox.internal",
"ssh_alias": "mesh-forge",
"ssh_port": "222",
"ssh_base": "ssh://git@novox.internal:222/",
"https_base": "https://git.novox.be/",
}
// forgeRender fills ${bound:forge-account:<key>} with the sample values, as the provisioner client does.
func forgeRender(t *testing.T, s string) string {
t.Helper()
for k, v := range forgeSample {
s = strings.ReplaceAll(s, "${bound:forge-account:"+k+"}", v)
}
if strings.Contains(s, "${") {
t.Errorf("unfilled placeholder in %q", s)
}
return s
}
// 1484/AC1: given sample bound values, the ssh config has Host mesh-forge, IdentitiesOnly yes and the key under the
// module's own home; known_hosts carries the host key; git's config rewrites https_base to ssh://mesh-forge/; no
// ${ is left.
func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) {
m := readForgeManifest(t)
cfg := forgeResource(t, m, "ssh-config")
want := "Host mesh-forge\n HostName novox.internal\n Port 222\n HostKeyAlias mesh-forge\n User git\n IdentityFile ~/.ssh/mesh_ed25519\n IdentitiesOnly yes\n"
if got := forgeRender(t, forgeTemplate(t, cfg)); got != want {
t.Errorf("ssh config = %q, want %q", got, want)
}
if cfg["path"] != "${dir:ssh-dir}/config" || cfg["mode"] != "0600" || cfg["into"] != nil {
t.Errorf("ssh config resource = %v, want the whole file ${dir:ssh-dir}/config at mode 0600", cfg)
}
kh := forgeResource(t, m, "known-hosts")
if got := strings.TrimSpace(forgeRender(t, forgeTemplate(t, kh))); got != "mesh-forge ssh-ed25519 AAAAC3Nza" {
t.Errorf("known_hosts = %q", got)
}
g := forgeResource(t, m, "git-config")
wantGit := "[url \"ssh://mesh-forge/\"]\n\tinsteadOf = https://git.novox.be/\n"
if got := forgeRender(t, forgeTemplate(t, g)); got != wantGit {
t.Errorf("git config = %q, want %q", got, wantGit)
}
dir := forgeResource(t, m, "ssh-dir")
if dir["mode"] != "0700" || dir["path"] != "${dir:state}/.ssh" {
t.Errorf("ssh-dir = %v, want ${dir:state}/.ssh at mode 0700", dir)
}
}
// 1484/AC2: the module requires forge-account, keeps the state keys, and every file resource takes its text from a
// template in the files/ archive (novox/hq ADR 0349 §2).
func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) {
m := readForgeManifest(t)
if !slices.Contains(m.Requires, any("forge-account")) {
t.Errorf("requires = %v, want forge-account among them", m.Requires)
}
keeps := false
for _, s := range m.State {
if s == "keys" {
keeps = true
}
}
if !keeps {
t.Errorf("state = %v lacks keys", m.State)
}
files := 0
for _, r := range m.Resources {
if _, inline := r["content"]; inline {
t.Errorf("resource %v keeps its text inline", r["id"])
}
if r["type"] == "file" {
files++
forgeTemplate(t, r)
}
}
if files == 0 {
t.Errorf("no file resource: the forge's ssh files are missing")
}
packs := 0
for _, a := range m.Build.Artifacts {
if a["kind"] == "archive" && a["from"] == "files" && a["name"] == "files" {
packs++
}
}
if packs != 1 {
t.Errorf("the build packs files/ into %d archives named files, not one", packs)
}
}
// forgeEd25519Line is a public key line with a well-formed ed25519 blob.
func forgeEd25519Line(comment string) string {
blob := []byte{}
for _, part := range [][]byte{[]byte("ssh-ed25519"), make([]byte, 32)} {
l := make([]byte, 4)
binary.BigEndian.PutUint32(l, uint32(len(part)))
blob = append(append(blob, l...), part...)
}
return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(blob) + " " + comment
}
type forgeKeys struct{ puts map[string][]byte }
func (f *forgeKeys) Put(key string, value any) (uint64, error) {
raw, _ := json.Marshal(value)
if f.puts == nil {
f.puts = map[string][]byte{}
}
f.puts[key] = raw
return uint64(len(f.puts)), nil
}
// forgeKeygen stands in for ssh-keygen: it writes the pair the command names.
func forgeKeygen(_ context.Context, _ []string, name string, args ...string) forgekey.Ran {
var file, comment string
for i, a := range args {
switch a {
case "-f":
file = args[i+1]
case "-C":
comment = args[i+1]
}
}
if name != "ssh-keygen" || file == "" {
return forgekey.Ran{Status: 1, Stderr: "unexpected"}
}
os.WriteFile(file, []byte("PRIVATE\n"), 0o600)
os.WriteFile(file+".pub", []byte(forgeEd25519Line(comment)+"\n"), 0o644)
return forgekey.Ran{}
}
// 1484/AC3: given the bundle started with an empty home, the keeper writes <node>.<module> with kind builder, and
// the key is in the home's .ssh.
func TestTheKeeperWritesTheModulesEntryWithKindBuilder(t *testing.T) {
home, st := t.TempDir(), &forgeKeys{}
if err := keeperFor(home, "ace", forgeModule, forgeKeygen, st).Once(context.Background()); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(home, ".ssh", "mesh_ed25519")); err != nil {
t.Errorf("no key in an empty home: %v", err)
}
raw, ok := st.puts["ace."+forgeModule]
if !ok || len(st.puts) != 1 {
t.Fatalf("entries = %v, want one at ace.%s", st.puts, forgeModule)
}
var e forgekey.KeyEntry
if err := json.Unmarshal(raw, &e); err != nil {
t.Fatal(err)
}
if e.Kind != "builder" || e.Node != "ace" || e.Account != forgeModule || !strings.HasPrefix(e.Public, "ssh-ed25519 ") {
t.Errorf("entry = %s", raw)
}
if strings.Contains(strings.ToUpper(string(raw)), "PRIVATE") {
t.Errorf("the entry carries private material: %s", raw)
}
}
// The home the manifest places is the one the account is given, so the keeper's home and the templates' agree.
func TestTheAccountsHomeIsTheOneTheManifestNames(t *testing.T) {
m := readForgeManifest(t)
for _, r := range m.Resources {
if r["type"] == "user" && r["name"] == forgeModule && r["home"] != forgeHome {
t.Errorf("the account's home is %v, want %s", r["home"], forgeHome)
}
}
}
modules/mesh-build-node/cmd/mesh-build-node/harness_test.go
+118
View File
@@ -0,0 +1,118 @@
package main
import (
"context"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// A fake machine for the step tests: a toolchain directory laid out as `npm ci --no-bin-links` leaves it, a work
// directory, a clock that only the fake commands advance, and a runner that records every command and answers by
// what the test set.
type call struct {
dir string // env -C <dir>
name string // the program after env -C <dir>
args []string // its arguments
env []string
}
type fakeMachine struct {
mu sync.Mutex
calls []call
now time.Time
// cost is how long a program takes, by the base name of its path; answer is what it prints and its status;
// effect runs the program's effects on the disk.
cost map[string]time.Duration
answer map[string]forgekey.Ran
effect func(c call)
}
func (f *fakeMachine) run(_ context.Context, env []string, name string, args ...string) forgekey.Ran {
f.mu.Lock()
defer f.mu.Unlock()
c := call{name: name, args: args, env: env}
key := filepath.Base(name)
if name == "env" && len(args) >= 3 && args[0] == "-C" {
c = call{dir: args[1], name: args[2], args: args[3:], env: env}
key = filepath.Base(c.name)
if key == "node" && len(c.args) > 0 {
key = filepath.Base(c.args[0]) // node <tsc>
}
} else if name == "git" {
for _, a := range args {
if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files"}, a) {
key = "git " + a
break
}
}
}
f.calls = append(f.calls, c)
f.now = f.now.Add(f.cost[key])
if f.effect != nil {
f.effect(c)
}
return f.answer[key]
}
func (f *fakeMachine) clock() time.Time { f.mu.Lock(); defer f.mu.Unlock(); return f.now }
// keys of the programs: tsc, esbuild, npm.
func newMachine(t *testing.T) (*Module, *fakeMachine, string) {
t.Helper()
home, cache, work := t.TempDir(), t.TempDir(), t.TempDir()
f := &fakeMachine{now: time.Unix(1_000_000, 0), cost: map[string]time.Duration{}, answer: map[string]forgekey.Ran{}}
m := &Module{Node: "ace", Recipe: Recipe{Steps: "1", Node: "v26.10.0", Lock: "lock"},
Toolchain: &Toolchain{Paths: Paths{Home: home, Cache: cache}, Registry: func() (Registry, error) {
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
}},
Work: work, Run: f.run, Clock: f.clock, Ctx: context.Background()}
// The toolchain directory as npm ci leaves it, with no bin links.
nm := filepath.Join(m.toolchainDir(), "node_modules")
for _, p := range []string{"typescript/bin/tsc", "esbuild/bin/esbuild", "@esbuild/linux-x64/bin/esbuild",
"@novox/mesh-sdk/dist/stdio.js", "@types/node/index.d.ts", "nats/lib/nats.js"} {
write(t, filepath.Join(nm, p), "x", 0o644)
}
return m, f, work
}
func write(t *testing.T, path, body string, mode os.FileMode) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(body), mode); err != nil {
t.Fatal(err)
}
}
// workspaceWith makes workspace "w1" of the machine with the files given (path under tree -> body).
func workspaceWith(t *testing.T, work string, files map[string]string) string {
t.Helper()
for p, body := range files {
write(t, filepath.Join(work, "w1", "tree", filepath.FromSlash(p)), body, 0o644)
}
return "w1"
}
func symlinksUnder(t *testing.T, root string) []string {
t.Helper()
var out []string
filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err == nil && info.Mode()&os.ModeSymlink != 0 {
out = append(out, p)
}
return nil
})
return out
}
func joined(c call) string { return c.name + " " + strings.Join(c.args, " ") }
modules/mesh-build-node/cmd/mesh-build-node/keys.go
+42
View File
@@ -0,0 +1,42 @@
package main
// The module's own forge key (novox/hq ADR 0466 §2, issue 1484): mesh-sdk's key keeper, kind builder, for the
// module's account in its home. It makes ~/.ssh/mesh_ed25519 when it is missing and writes one entry to this
// module's `keys` state at <node>.<account>, which gitea's provider reads to register the key on the forge user
// that reads and pushes nothing (ADR 0466 §3).
import (
"context"
"fmt"
"os"
"os/user"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// keeperFor is this module's keeper: go/forgekey's, of kind builder, for account in home on node, whose entry lies
// at <node>.<account> of the keys state.
func keeperFor(home, node, account string, run forgekey.Runner, state forgekey.KeyPutter) *forgekey.KeyKeeper {
return &forgekey.KeyKeeper{Home: home, Node: node, Account: account, Kind: forgekey.KindBuilder, Run: run, State: state, Now: time.Now}
}
// startKeeper runs the keeper when the runtime launched this process for a node: outside a launch — a build, a
// check — there is no state to put the entry in.
func startKeeper(ctx context.Context) {
node := strings.TrimSpace(os.Getenv("MESH_NODE"))
u, err := user.Current()
if node == "" || err != nil || u.HomeDir == "" {
return
}
k := keeperFor(u.HomeDir, node, u.Username, forgekey.ExecRunner, keysState{})
k.Keep(ctx, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-node] "+f+"\n", a...) })
}
type keysState struct{}
func (keysState) Put(key string, value any) (uint64, error) {
return stdio.State("keys").Put(key, value)
}
modules/mesh-build-node/cmd/mesh-build-node/main.go
+62
View File
@@ -0,0 +1,62 @@
// mesh-build-node: the Node and TypeScript build module (novox/hq ADR 0405, ADR 0466 §1, ADR 0469, ADR 0470; design
// 47 R4c-3, issue 1488). A Go tools bundle the node's runtime launches as the module's own account (runs-as). It names
// the one Node version it builds with, installs the TypeScript toolchain its own lock names, writes the Node version
// it finds to <node>.typescript of its `toolchains` state at start and before every step, and refuses every step on
// a node that has another: its health is then unhealthy, so the controller raises the node's condition. What it
// says, it says on stderr; stdout is the MCP channel.
package main
import (
"context"
"fmt"
"os"
"os/user"
"path/filepath"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// CacheDir is the cache directory, outside the account's home (ADR 0466 §1; module.json resource `cache`).
const CacheDir = "/var/lib/mesh-build-node-cache"
// WorkDir is the work directory, outside the home (module.json resource `work`).
const WorkDir = "/var/lib/mesh-build-node-work"
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-node] "+format+"\n", a...) }
func main() {
ctx := context.Background()
home := ""
if u, err := user.Current(); err == nil {
home = u.HomeDir
}
paths := Paths{Home: home, Cache: CacheDir}
registry := func() (Registry, error) {
return ReadRegistry(filepath.Join(home, "package-registry.json"), filepath.Join(home, "package-registry.secret"))
}
m := &Module{
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
Reload: func() (Recipe, error) { return ReadRecipe(paths.LockPath()) },
Probe: ProbeNode,
Toolchain: &Toolchain{Paths: paths, Run: forgekey.ExecRunnerClean(0), Registry: registry},
Toolchains: stdio.State(delivery.ToolchainsState).Put,
Emit: func(t string, b any) error { return stdio.Emit(t, b) },
Steps: stdio.State(delivery.StepsState).Put,
Work: WorkDir,
Run: forgekey.ExecRunnerClean(0),
Ctx: ctx,
Say: logf,
}
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
if m.Node != "" {
go m.Start(ctx)
}
go startKeeper(ctx)
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
logf("%v", err)
os.Exit(1)
}
}
modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go
+193
View File
@@ -0,0 +1,193 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"regexp"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The catalogue's root and this module's directory.
const (
catalogueRoot = "../../../.."
moduleDir = "../.."
)
func readManifest(t *testing.T) catalogue.Manifest {
t.Helper()
m, err := catalogue.ReadTreeManifest(catalogueRoot, moduleDir)
if err != nil {
t.Fatalf("mesh-sdk's validator refuses module.json: %v", err)
}
return m
}
func resource(t *testing.T, m catalogue.Manifest, id string) map[string]any {
t.Helper()
for _, r := range m.Resources {
if r["id"] == id {
return r
}
}
t.Fatalf("module.json declares no resource %q", id)
return nil
}
// 1488/AC1: mesh-sdk's validator accepts the module (alone on a shelf too), and it runs as mesh-build-node with its
// home at mode 0700, requires user-namespaces and declares the four packages and carries toolchain/package-lock.json.
func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *testing.T) {
m := readManifest(t)
if problems := catalogue.CatalogueProblems(catalogue.Shelf{m.Module: m}); len(problems) > 0 {
t.Errorf("the catalogue's rules refuse it: %v", problems)
}
if m.Module != "mesh-build-node" || m.RunsAs != "mesh-build-node" {
t.Errorf("module %q runs as %q, want mesh-build-node as itself", m.Module, m.RunsAs)
}
account := resource(t, m, "account")
if account["type"] != "user" || account["name"] != "mesh-build-node" || account["shell"] != "/usr/bin/nologin" || account["home"] != "/var/lib/mesh-build-node" {
t.Errorf("account = %v", account)
}
home := resource(t, m, "state")
if home["type"] != "directory" || home["mode"] != "0700" || home["owner"] != "mesh-build-node" || home["place"] != "." {
t.Errorf("home = %v, want its directory at mode 0700 owned by the account", home)
}
if !slices.Contains(m.Capabilities, "user-namespaces") {
t.Errorf("capabilities = %v, want user-namespaces", m.Capabilities)
}
packages := []string{}
for _, r := range m.Resources {
if r["type"] == "package" {
packages = append(packages, r["package"].(string))
}
}
slices.Sort(packages)
if want := []string{"git", "nodejs", "npm", "openssh"}; !slices.Equal(packages, want) {
t.Errorf("packages = %v, want %v", packages, want)
}
}
// The work and cache directories are outside the home (ADR 0466 §1), owned by the account.
func TestWorkAndCacheDirectoriesLieOutsideTheHome(t *testing.T) {
m := readManifest(t)
for _, id := range []string{"work", "cache"} {
r := resource(t, m, id)
path, _ := r["path"].(string)
if path == "" || path == "/var/lib/mesh-build-node" || len(path) > len("/var/lib/mesh-build-node") && path[:len("/var/lib/mesh-build-node/")] == "/var/lib/mesh-build-node/" {
t.Errorf("%s is at %q, inside or at the home", id, path)
}
if r["owner"] != "mesh-build-node" {
t.Errorf("%s is owned by %v", id, r["owner"])
}
}
}
// The states are the three the module keeps, no seat is claimed (ADR 0469 §6), and the health is the module's own
// tool, which the manifest also lists among its tools.
func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) {
m := readManifest(t)
states := []string{}
for _, s := range m.State {
states = append(states, s.Name)
}
slices.Sort(states)
if want := []string{"keys", "steps", "toolchains"}; !slices.Equal(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if len(m.Claims) != 0 {
t.Errorf("claims = %v, want no seat", m.Claims)
}
if !slices.Contains(m.Tools, HealthTool) {
t.Errorf("tools = %v lack %s", m.Tools, HealthTool)
}
raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json"))
if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) {
t.Errorf("runtime-health does not name %s", HealthTool)
}
}
// Every tool the manifest names is served, and every tool served is named.
func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
m := readManifest(t)
served := []string{}
for _, tool := range (&Module{}).Tools((&Module{}).StepTools()) {
served = append(served, tool.Name)
}
slices.Sort(served)
named := slices.Clone(m.Tools)
slices.Sort(named)
if !slices.Equal(served, named) {
t.Errorf("served %v, manifest names %v", served, named)
}
}
// 1488/AC1: the module carries the committed lock beside its package.json, and the manifest unpacks them into the
// home's toolchain directory, where the program reads them.
func TestTheModuleCarriesItsToolchainLockAndTheManifestPlacesIt(t *testing.T) {
m := readManifest(t)
for _, name := range []string{"package.json", "package-lock.json"} {
if _, err := os.Stat(filepath.Join(moduleDir, "toolchain", name)); err != nil {
t.Errorf("the module carries no toolchain/%s: %v", name, err)
}
}
if !slices.Contains(m.Requires, "npm-package-registry") {
t.Errorf("requires = %v lack npm-package-registry, which the install resolves @novox from", m.Requires)
}
}
// 1488/AC1: the lock names the versions of the recipe: typescript 5.9.3, esbuild 0.25.12, @novox/mesh-sdk 0.1.14
// and nats, all by the lock's own hash (read from the file the module carries).
func TestTheLockNamesTheToolchainVersions(t *testing.T) {
raw, err := os.ReadFile(filepath.Join(moduleDir, "toolchain", "package-lock.json"))
if err != nil {
t.Fatal(err)
}
var lock struct {
Packages map[string]struct {
Version string `json:"version"`
} `json:"packages"`
}
if err := json.Unmarshal(raw, &lock); err != nil {
t.Fatal(err)
}
for pkg, want := range map[string]string{"typescript": "5.9.3", "esbuild": "0.25.12", "@novox/mesh-sdk": "0.1.14", "nats": "2.29.3", "@types/node": "22.20.1"} {
if got := lock.Packages["node_modules/"+pkg].Version; got != want {
t.Errorf("%s = %q in the lock, want %s", pkg, got, want)
}
}
}
// 1488 (the catalogue's manifest check): a manifest names no host, the mesh being name-agnostic. The tree-read
// manifest carries each template's text as the resource's content, and the toolchain's lock names registries, so it is
// shipped as an archive artifact instead, never as a template. No resource of the module, read as the mesh reads it,
// carries content naming a host, and the raw module.json names none either.
func TestNoResourceCarriesContentNamingAHost(t *testing.T) {
host := regexp.MustCompile(`(?i)://[a-z0-9]|\b[a-z0-9-]+(\.[a-z0-9-]+)*\.(com|org|net|be|io|dev)\b`)
m := readManifest(t)
for i, r := range m.Resources {
if c, _ := r["content"].(string); host.MatchString(c) {
t.Errorf("resources[%d] (%v) carries content naming %q", i, r["id"], host.FindString(c))
}
}
raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json"))
if h := host.FindString(string(raw)); h != "" {
t.Errorf("module.json names %q", h)
}
}
// 1488: the toolchain's package.json and lock are shipped in the archive artifact `toolchain` and unpacked into the
// home's toolchain directory, where the program reads them.
func TestTheToolchainFilesAreAnArchiveUnpackedIntoTheHome(t *testing.T) {
m := readManifest(t)
r := resource(t, m, "toolchain-dir")
if r["type"] != "archive" || r["path"] != "${dir:state}/toolchain" || r["owner"] != "mesh-build-node" {
t.Errorf("toolchain-dir = %v, want an archive unpacked at ${dir:state}/toolchain", r)
}
if _, err := os.Stat(filepath.Join(moduleDir, "toolchain", "package-lock.json")); err != nil {
t.Errorf("the module carries no toolchain/package-lock.json: %v", err)
}
}
modules/mesh-build-node/cmd/mesh-build-node/module.go
+199
View File
@@ -0,0 +1,199 @@
package main
import (
"context"
"fmt"
"os"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
"git.novox.be/novox/mesh-sdk/go/health"
)
// CheckRound is how often the module reads the installed toolchain again, beside the check before every step and
// the one every health look makes.
const CheckRound = 10 * time.Minute
// Module is the Node build module's one running state: the node it runs on, its recipe, the toolchain guard, the
// toolchain directory and the states it writes (toolchains, steps).
type Module struct {
Node string
// Recipe is the recipe in force. Reload, when set, reads it again at every check (the lock file the mesh
// writes can change under a running module); a failure to read it is a refusal.
Recipe Recipe
Reload func() (Recipe, error)
// Probe reads the installed Node version (ProbeNode).
Probe func(ctx context.Context) (string, error)
// Toolchain installs the toolchain directory; nil in a module that only guards.
Toolchain *Toolchain
// Work is the work directory, outside the home: a workspace is <Work>/<step id>/tree. Run runs the commands of a
// step (a fake in a test). Clock is the time the steps are timed by.
Work string
Run forgekey.Runner
Clock func() time.Time
// Toolchains and Steps are the Put of the states of those names.
Toolchains, Steps buildstep.Put
// Emit tells the bus an event (stdio.Emit); nil emits nothing. The guard emits toolchain.reported after each write of
// the toolchains key, so the indexer, which holds no timer, learns that the bucket exists (issue 1444).
Emit func(eventType string, body any) error
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
Ctx context.Context
Say func(format string, a ...any)
mu sync.Mutex // guards Recipe
install sync.Mutex // one install at a time
}
func (m *Module) recipe() Recipe {
m.mu.Lock()
defer m.mu.Unlock()
return m.Recipe
}
func (m *Module) guard(r Recipe) buildstep.Guard {
return buildstep.Guard{Language: "typescript", Want: r.Node, Recipe: r.Version(), Probe: m.Probe, Node: m.Node, Put: m.reportingPut()}
}
// Check reads the installed toolchain, writes <node>.typescript of the toolchains state and answers nil, or the
// refusal (buildstep.Guard.Check). It runs at start, before every step, at every health look and every CheckRound.
func (m *Module) Check(ctx context.Context) error {
if m.Reload != nil {
r, err := m.Reload()
if err != nil {
return &buildstep.Refusal{Text: err.Error(), Want: RecipeNode}
}
m.mu.Lock()
m.Recipe = r
m.mu.Unlock()
}
return m.guard(m.recipe()).Check(ctx)
}
// Prepare is Check and then the toolchain directory of the recipe in force: what every step runs behind.
func (m *Module) Prepare(ctx context.Context) error {
if err := m.Check(ctx); err != nil {
return err
}
if m.Toolchain == nil {
return nil
}
m.install.Lock()
defer m.install.Unlock()
return m.Toolchain.Ensure(ctx, m.recipe().Version())
}
// Start prepares once, as the module starts, and then every CheckRound until ctx ends.
func (m *Module) Start(ctx context.Context) {
for {
if err := m.Prepare(ctx); err != nil && m.Say != nil {
m.Say("the toolchain: %v", err)
}
select {
case <-ctx.Done():
return
case <-time.After(CheckRound):
}
}
}
// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy while the toolchain installed is not
// the recipe's, saying both versions, so the controller raises the node's condition. It looks again each time it is
// asked.
func (m *Module) Health(ctx context.Context) health.Answer {
if err := m.Check(ctx); err != nil {
return health.Unhealthy(err.Error())
}
r := m.recipe()
return health.Healthy(fmt.Sprintf("node %s installed, as the recipe %s builds with", r.Node, r.Version()))
}
// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on
// its key in the steps state. The guard runs inside the step, before run, so a node that refuses ends every step
// refused with both versions named.
func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
return func(args map[string]any) (any, error) {
id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) {
if err := m.Prepare(ctx); err != nil {
return nil, err
}
return run(ctx, args)
})
if err != nil {
return nil, err
}
key, _ := delivery.StepKey(m.Node, id)
return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil
}
}
// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error.
func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
return func(args map[string]any) (any, error) {
if err := m.Prepare(m.Ctx); err != nil {
return nil, err
}
return run(m.Ctx, args)
}
}
// Status is what the module reports of itself: the node, the recipe, the toolchain found and why it refuses, if it
// does.
func (m *Module) Status(ctx context.Context) map[string]any {
err := m.Check(ctx)
installed, _ := m.Probe(ctx)
r := m.recipe()
out := map[string]any{"node": m.Node, "recipe": r.Version(), "node-version": r.Node, "installed": installed, "refusing": err != nil}
if m.Toolchain != nil {
out["toolchain-dir"] = m.Toolchain.Paths.Dir(r.Version())
out["toolchain-installed"] = m.Toolchain.Present(r.Version())
}
if err != nil {
out["why"] = err.Error()
}
return out
}
func (m *Module) now() time.Time {
if m.Clock != nil {
return m.Clock()
}
return time.Now()
}
// defaultPath is where a step looks for node, npm, git and ssh when this process was given no PATH.
const defaultPath = "/usr/local/bin:/usr/bin:/bin"
// baseEnv is the environment of a step's command beyond what the step adds: PATH, the account's own HOME (its ssh
// files and git configuration) and a fixed locale. A command is given these and its own words, never this process's
// environment, which holds the module's bus credentials (ADR 0466 §4).
func (m *Module) baseEnv() []string {
return []string{"PATH=" + pathOrDefault(), "HOME=" + m.Toolchain.homeOrEmpty(), "LC_ALL=C"}
}
func pathOrDefault() string {
if path := os.Getenv("PATH"); path != "" {
return path
}
return defaultPath
}
// ToolchainReported is the event the guard emits after it writes the toolchains key.
const ToolchainReported = "toolchain.reported"
// reportingPut is the toolchains Put that then emits toolchain.reported with the node and the language. A write that
// failed emits nothing; an emit that failed is said and does not refuse the step.
func (m *Module) reportingPut() buildstep.Put {
return func(key string, value any) (uint64, error) {
rev, err := m.Toolchains(key, value)
if err != nil || m.Emit == nil {
return rev, err
}
if eerr := m.Emit(ToolchainReported, map[string]string{"node": m.Node, "language": "typescript"}); eerr != nil && m.Say != nil {
m.Say("the event %s could not be emitted: %v", ToolchainReported, eerr)
}
return rev, nil
}
}
modules/mesh-build-node/cmd/mesh-build-node/module_test.go
+231
View File
@@ -0,0 +1,231 @@
package main
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// keptStates stands in for the toolchains and the steps states.
type keptStates struct {
mu sync.Mutex
toolchains map[string]delivery.Toolchain
steps map[string][]delivery.Step
ended chan string
}
func newStates() *keptStates {
return &keptStates{toolchains: map[string]delivery.Toolchain{}, steps: map[string][]delivery.Step{}, ended: make(chan string, 16)}
}
func (s *keptStates) putToolchain(key string, value any) (uint64, error) {
tc, err := delivery.DecodeToolchain(value.(json.RawMessage))
if err != nil {
return 0, err
}
s.mu.Lock()
defer s.mu.Unlock()
s.toolchains[key] = tc
return 1, nil
}
func (s *keptStates) putStep(key string, value any) (uint64, error) {
st, err := delivery.DecodeStep(value.(json.RawMessage))
if err != nil {
return 0, err
}
s.mu.Lock()
s.steps[key] = append(s.steps[key], st)
s.mu.Unlock()
if st.State != delivery.StepRunning {
s.ended <- key
}
return 1, nil
}
func (s *keptStates) toolchain(key string) (delivery.Toolchain, bool) {
s.mu.Lock()
defer s.mu.Unlock()
tc, ok := s.toolchains[key]
return tc, ok
}
func (s *keptStates) waitEnd(t *testing.T) delivery.Step {
t.Helper()
select {
case key := <-s.ended:
s.mu.Lock()
defer s.mu.Unlock()
v := s.steps[key]
return v[len(v)-1]
case <-time.After(3 * time.Second):
t.Fatal("the step never ended on its key")
return delivery.Step{}
}
}
func moduleWith(states *keptStates, installed string) *Module {
return &Module{Node: "ace", Recipe: Recipe{Steps: "1", Node: "v26.10.0", Lock: "lock"},
Probe: func(context.Context) (string, error) { return installed, nil },
Toolchains: states.putToolchain, Steps: states.putStep, Ctx: context.Background()}
}
// 1488/AC3: a probe answering v26.10.0: the bundle's start writes ace.typescript with that version and the recipe's,
// and the health tool answers healthy.
func TestStartWritesTheToolchainAndHealthIsHealthy(t *testing.T) {
states := newStates()
m := moduleWith(states, "v26.10.0")
ctx, stop := context.WithCancel(context.Background())
defer stop()
go m.Start(ctx)
var tc delivery.Toolchain
for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
if got, ok := states.toolchain("ace.typescript"); ok {
tc = got
break
}
}
if tc.Version != "v26.10.0" || tc.Recipe != m.Recipe.Version() || tc.Language != "typescript" {
t.Fatalf("ace.typescript = %+v, want v26.10.0 and recipe %s", tc, m.Recipe.Version())
}
if a := m.Health(ctx); a.State != health.StateHealthy || !a.Healthy {
t.Errorf("health = %+v, want healthy", a)
}
}
// 1488/AC3: a probe answering v26.11.0: every step tool's step ends refused naming both versions, ace.typescript
// holds v26.11.0, and the health tool answers unhealthy naming both.
func TestAnotherNodeRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
states := newStates()
m := moduleWith(states, "v26.11.0")
ran := false
run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
steps := append([]StepTool{
{Name: "long_probe", Long: true, Run: run},
{Name: "short_probe", Run: run},
}, (&Module{}).StepTools()...)
for _, tool := range m.Tools(steps) {
if tool.Name == StatusTool || tool.Name == HealthTool {
continue
}
var reason string
out, err := tool.Run(map[string]any{})
if isLong(steps, tool.Name) {
if err != nil {
t.Fatalf("%s: %v", tool.Name, err)
}
if out.(map[string]any)["step"] == "" {
t.Fatalf("%s answered no step id: %v", tool.Name, out)
}
st := states.waitEnd(t)
if st.State != delivery.StepRefused {
t.Errorf("%s: the step ended %s, want refused", tool.Name, st.State)
}
reason = st.Reason
} else {
if err == nil {
t.Fatalf("%s: answered %v, want a refusal", tool.Name, out)
}
reason = err.Error()
}
if !strings.Contains(reason, "v26.11.0") || !strings.Contains(reason, "v26.10.0") {
t.Errorf("%s: the refusal %q does not name both versions", tool.Name, reason)
}
}
if ran {
t.Errorf("a step ran on a node that refuses")
}
if tc, _ := states.toolchain("ace.typescript"); tc.Version != "v26.11.0" {
t.Errorf("ace.typescript holds %q, want the installed v26.11.0", tc.Version)
}
a := m.Health(context.Background())
if a.State != health.StateUnhealthy || !strings.Contains(a.Why, "v26.11.0") || !strings.Contains(a.Why, "v26.10.0") {
t.Errorf("health = %+v, want unhealthy naming both versions", a)
}
}
func isLong(steps []StepTool, name string) bool {
for _, s := range steps {
if s.Name == name {
return s.Long
}
}
return false
}
// 1488/AC2: the recipe version moves with one byte of the lock and with the Node version, and with the recipe's own
// version; equal recipes have the same one.
func TestRecipeVersionMovesWithEachOfItsValues(t *testing.T) {
lock := []byte(`{"lockfileVersion":3}`)
base := Recipe{Steps: "1", Node: "v26.10.0", Lock: LockHash(lock)}
if base.Version() != (Recipe{Steps: "1", Node: "v26.10.0", Lock: LockHash(lock)}).Version() {
t.Errorf("one recipe has two versions")
}
seen := map[string]string{base.Version(): "base"}
for name, r := range map[string]Recipe{
"one byte of the lock": {Steps: "1", Node: "v26.10.0", Lock: LockHash([]byte(`{"lockfileVersion":4}`))},
"node": {Steps: "1", Node: "v26.11.0", Lock: LockHash(lock)},
"steps": {Steps: "2", Node: "v26.10.0", Lock: LockHash(lock)},
} {
v := r.Version()
if was, dup := seen[v]; dup {
t.Errorf("the recipe differing in %s has the version of the %s recipe: %s", name, was, v)
}
seen[v] = name
}
}
// 1488/AC2: the recipe reads the lock file's bytes: a lock changed on disk is another recipe version, a lock missing
// is an error.
func TestReadRecipeHashesTheLockFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "package-lock.json")
if _, err := ReadRecipe(path); err == nil {
t.Errorf("a missing lock was accepted")
}
os.WriteFile(path, []byte("a"), 0o600)
a, err := ReadRecipe(path)
if err != nil || a.Node != "v26.10.0" {
t.Fatalf("= %+v, %v", a, err)
}
os.WriteFile(path, []byte("b"), 0o600)
b, _ := ReadRecipe(path)
if a.Version() == b.Version() {
t.Errorf("one byte of the lock did not move the recipe version")
}
}
// 1488/AC3: a lock changed under a running module is read at the next check, and a lock that cannot be read refuses.
func TestACheckReadsTheRecipeAgain(t *testing.T) {
states := newStates()
m := moduleWith(states, "v26.10.0")
r2 := Recipe{Steps: "1", Node: "v26.10.0", Lock: "other"}
m.Reload = func() (Recipe, error) { return r2, nil }
if err := m.Check(context.Background()); err != nil {
t.Fatal(err)
}
if tc, _ := states.toolchain("ace.typescript"); tc.Recipe != r2.Version() {
t.Errorf("ace.typescript holds recipe %s, want the reloaded %s", tc.Recipe, r2.Version())
}
m.Reload = func() (Recipe, error) { return Recipe{}, errors.New("no lock") }
if err := m.Check(context.Background()); err == nil || m.Health(context.Background()).Healthy {
t.Errorf("a lock that cannot be read neither refused nor made the module unhealthy: %v", err)
}
}
func TestParseNodeVersion(t *testing.T) {
if v, err := parseNodeVersion("v26.10.0\n"); err != nil || v != "v26.10.0" {
t.Errorf("= %q, %v", v, err)
}
if _, err := parseNodeVersion("not node"); err == nil {
t.Errorf("accepted a text that is not `node --version`'s")
}
}
modules/mesh-build-node/cmd/mesh-build-node/probe.go
+31
View File
@@ -0,0 +1,31 @@
package main
import (
"context"
"fmt"
"os/exec"
"strings"
"time"
)
// ProbeTimeout bounds `node --version`.
const ProbeTimeout = 15 * time.Second
// ProbeNode reads the installed Node version: what `node --version` prints ("v26.10.0"), as RecipeNode is written.
func ProbeNode(ctx context.Context) (string, error) {
ctx, cancel := context.WithTimeout(ctx, ProbeTimeout)
defer cancel()
out, err := exec.CommandContext(ctx, "node", "--version").Output()
if err != nil {
return "", err
}
return parseNodeVersion(string(out))
}
func parseNodeVersion(out string) (string, error) {
v := strings.TrimSpace(out)
if len(v) < 2 || v[0] != 'v' || v[1] < '0' || v[1] > '9' || strings.ContainsAny(v, " \n\t") {
return "", fmt.Errorf("`node --version` answered %q", v)
}
return v, nil
}
modules/mesh-build-node/cmd/mesh-build-node/recipe.go
+58
View File
@@ -0,0 +1,58 @@
package main
// The recipe of the Node build module (novox/hq ADR 0405 §1 and §7, ADR 0469 §1 and §4; issue 1488): the one Node
// version it builds with, and the recipe's own version, a hash of the recipe's version, that Node version and the
// bytes of the lock file that names the TypeScript toolchain. A node whose installed Node differs from the recipe's
// refuses every step (module.go), so all builders run this version or none builds.
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
)
// RecipeNode is the Node version the recipe builds with: the text `node --version` prints for it, which is what the
// guard compares and what the indexer reads from <node>.typescript of the toolchains state.
const RecipeNode = "v26.10.0"
// RecipeSteps is the recipe's own version: changed whenever the steps of fetch, test or build change in a way that
// can change what they produce (ADR 0405 §7). It is one of the three values the recipe version hashes.
const RecipeSteps = "1"
// Recipe is every value that makes up the recipe version.
type Recipe struct {
// Steps is RecipeSteps.
Steps string
// Node is the toolchain's version as `node --version` prints it.
Node string
// Lock is the hex sha256 of the bytes of toolchain/package-lock.json (LockHash).
Lock string
}
// LockHash is the text a lock file's bytes stand for in the recipe version.
func LockHash(lock []byte) string {
sum := sha256.Sum256(lock)
return hex.EncodeToString(sum[:])
}
// ReadRecipe is the recipe of this build of the module with the lock file read from path. A missing lock is an
// error: with no lock there is no toolchain to name.
func ReadRecipe(lockPath string) (Recipe, error) {
raw, err := os.ReadFile(lockPath)
if err != nil {
return Recipe{}, fmt.Errorf("the toolchain's lock file cannot be read: %w", err)
}
return Recipe{Steps: RecipeSteps, Node: RecipeNode, Lock: LockHash(raw)}, nil
}
// Version is the recipe version: a hash of the recipe's own version, the Node version and the lock file's hash, so a
// change of any of the three moves it and with it every fingerprint that holds it (ADR 0405 §7, ADR 0465 §2). The
// fields are named in the hashed text, so no two recipes share a text.
func (r Recipe) Version() string {
h := sha256.New()
for _, f := range [][2]string{{"steps", r.Steps}, {"node", r.Node}, {"lock", r.Lock}} {
h.Write([]byte(f[0] + "=" + f[1] + "\n"))
}
return hex.EncodeToString(h.Sum(nil))[:16]
}
modules/mesh-build-node/cmd/mesh-build-node/registry.go
+81
View File
@@ -0,0 +1,81 @@
package main
// How the toolchain's install reaches the mesh's package registry (novox/hq ADR 0405 §5, ADR 0336; issue 1488).
// The lock names @novox/mesh-sdk at the registry's public address, and npm installs a locked tarball only from the
// registry it is configured with, so the install is told that registry for the @novox scope and nothing else: every
// other package resolves from the public one. This is how today's build agent names it (mesh-controller
// internal/builder/dependencies.go: the scope's registry from the npm-package-registry binding, the account and
// secret the mesh sealed), read here from the module's own binding and secret files (module.json `binds` and
// `secrets`), the same files the lab module reads.
import (
"encoding/base64"
"encoding/json"
"fmt"
"os"
"strings"
)
// Scope is the npm scope the mesh's registry answers for.
const Scope = "@novox"
// Registry is the mesh's package registry as the binding names it.
type Registry struct {
URL, Username, Password string
}
// ReadRegistry reads the npm-package-registry binding and its sealed secret. The registry is the binding's
// `public-root` when it has one (the address the lock holds), and otherwise scheme://at:port plus `npm-path`.
func ReadRegistry(bindingPath, secretPath string) (Registry, error) {
raw, err := os.ReadFile(bindingPath)
if err != nil {
return Registry{}, fmt.Errorf("no package registry is bound to this module: %w", err)
}
var b struct {
At string `json:"at"`
As string `json:"as"`
Serves struct {
Scheme string `json:"scheme"`
Port int `json:"port"`
NpmPath string `json:"npm-path"`
PublicRoot string `json:"public-root"`
} `json:"serves"`
}
if err := json.Unmarshal(raw, &b); err != nil {
return Registry{}, fmt.Errorf("the package registry binding is not JSON: %w", err)
}
url := b.Serves.PublicRoot
if url == "" && b.At != "" && b.Serves.NpmPath != "" {
scheme := b.Serves.Scheme
if scheme == "" {
scheme = "http"
}
url = fmt.Sprintf("%s://%s:%d%s", scheme, b.At, b.Serves.Port, b.Serves.NpmPath)
}
if url == "" {
return Registry{}, fmt.Errorf("the package registry binding names no address")
}
if !strings.HasSuffix(url, "/") {
url += "/"
}
secret, err := os.ReadFile(secretPath)
if err != nil {
return Registry{}, fmt.Errorf("the package registry's secret cannot be read: %w", err)
}
return Registry{URL: url, Username: b.As, Password: strings.TrimSpace(string(secret))}, nil
}
// Npmrc is the npm configuration text for the install: the @novox scope at the registry, with the account's
// credential for it when there is one.
func (r Registry) Npmrc() string {
var sb strings.Builder
fmt.Fprintf(&sb, "%s:registry=%s\n", Scope, r.URL)
if r.Username != "" && r.Password != "" {
hostPath := r.URL
if i := strings.Index(hostPath, "://"); i >= 0 {
hostPath = hostPath[i+1:]
}
fmt.Fprintf(&sb, "%s:_auth=%s\n", hostPath, base64.StdEncoding.EncodeToString([]byte(r.Username+":"+r.Password)))
}
return sb.String()
}
modules/mesh-build-node/cmd/mesh-build-node/toolchain.go
+106
View File
@@ -0,0 +1,106 @@
package main
// The toolchain directory (novox/hq ADR 0405 §5, ADR 0469 §4; issue 1488): typescript, esbuild, @novox/mesh-sdk and
// nats by the versions the module's own lock names, installed by `npm ci --ignore-scripts` into
// <cache dir>/toolchain/<recipe version> when that directory is missing. No bin links are made (--no-bin-links): the
// toolchain holds no symlink, and tsc and esbuild are run by their real paths. A change of the lock, of the Node
// version or of the recipe moves the recipe version and so installs a fresh directory beside the old one; steps use the one named
// by the current recipe.
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// installedMark is written last into a toolchain directory, so a directory an install did not finish is not taken
// for a toolchain.
const installedMark = ".installed"
// Paths are the module's places on its machine.
type Paths struct {
// Home is the account's home (the state directory): where the mesh writes toolchain/package.json and
// toolchain/package-lock.json (module.json resources) and the registry binding and secret.
Home string
// Cache is the cache directory, outside the home (ADR 0466 §1).
Cache string
}
// LockPath is the committed lock file the mesh writes into the home.
func (p Paths) LockPath() string { return filepath.Join(p.Home, "toolchain", "package-lock.json") }
// Dir is the toolchain directory of a recipe version.
func (p Paths) Dir(version string) string { return filepath.Join(p.Cache, "toolchain", version) }
// Toolchain installs the toolchain directories.
type Toolchain struct {
Paths Paths
Run forgekey.Runner
Registry func() (Registry, error)
}
// Present reports whether the recipe version's toolchain directory is installed.
func (t Toolchain) Present(version string) bool {
_, err := os.Stat(filepath.Join(t.Paths.Dir(version), installedMark))
return err == nil
}
// Ensure installs the recipe version's toolchain directory unless it is there: the module's package.json and lock
// are copied in, and `npm ci --ignore-scripts` runs once, the mesh's registry named for the @novox scope. A directory
// an earlier install left unfinished is removed first.
func (t Toolchain) Ensure(ctx context.Context, version string) error {
if t.Present(version) {
return nil
}
dir := t.Paths.Dir(version)
if err := os.RemoveAll(dir); err != nil {
return err
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return err
}
for _, name := range []string{"package.json", "package-lock.json"} {
raw, err := os.ReadFile(filepath.Join(t.Paths.Home, "toolchain", name))
if err != nil {
return fmt.Errorf("the toolchain's %s cannot be read: %w", name, err)
}
if err := os.WriteFile(filepath.Join(dir, name), raw, 0o600); err != nil {
return err
}
}
reg, err := t.Registry()
if err != nil {
return err
}
npmrc := filepath.Join(dir, ".npmrc-install")
if err := os.WriteFile(npmrc, []byte(reg.Npmrc()), 0o600); err != nil {
return err
}
defer os.Remove(npmrc)
ran := t.Run(ctx, []string{"PATH=" + pathOrDefault(), "HOME=" + t.Paths.Home, "LC_ALL=C", "npm_config_cache=" + filepath.Join(t.Paths.Cache, "npm")},
"npm", "ci", "--ignore-scripts", "--no-bin-links", "--no-audit", "--no-fund", "--prefix", dir, "--userconfig", npmrc)
if ran.Err != "" || ran.Status != 0 {
return fmt.Errorf("npm ci --ignore-scripts for the toolchain %s failed (status %d %s): %s",
version, ran.Status, ran.Err, tail(ran.Stderr))
}
return os.WriteFile(filepath.Join(dir, installedMark), []byte(version+"\n"), 0o600)
}
func tail(s string) string {
s = strings.TrimSpace(s)
if len(s) > 400 {
s = s[len(s)-400:]
}
return s
}
func (t *Toolchain) homeOrEmpty() string {
if t == nil {
return ""
}
return t.Paths.Home
}
modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go
+173
View File
@@ -0,0 +1,173 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
type npmCall struct {
name string
args []string
env []string
}
type fakeNpm struct {
mu sync.Mutex
calls []npmCall
ran forgekey.Ran
}
func (f *fakeNpm) run(_ context.Context, env []string, name string, args ...string) forgekey.Ran {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, npmCall{name, args, env})
if f.ran.Status == 0 && f.ran.Err == "" {
// npm ci writes node_modules into --prefix.
for i, a := range args {
if a == "--prefix" {
os.MkdirAll(filepath.Join(args[i+1], "node_modules"), 0o700)
}
}
}
return f.ran
}
func toolchainIn(t *testing.T, fake *fakeNpm) Toolchain {
t.Helper()
home, cache := t.TempDir(), t.TempDir()
if err := os.MkdirAll(filepath.Join(home, "toolchain"), 0o700); err != nil {
t.Fatal(err)
}
os.WriteFile(filepath.Join(home, "toolchain", "package.json"), []byte(`{"name":"x"}`), 0o600)
os.WriteFile(filepath.Join(home, "toolchain", "package-lock.json"), []byte(`{"lock":1}`), 0o600)
return Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: fake.run,
Registry: func() (Registry, error) {
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil
}}
}
// 1488/AC4: given a missing toolchain directory, npm ci --ignore-scripts runs once in <cache>/toolchain/<recipe
// version> with the module's lock; asked again with the directory present, nothing runs.
func TestAMissingToolchainDirectoryIsInstalledOnceFromTheLock(t *testing.T) {
fake := &fakeNpm{}
tc := toolchainIn(t, fake)
version := Recipe{Steps: "1", Node: "v26.10.0", Lock: "x"}.Version()
dir := filepath.Join(tc.Paths.Cache, "toolchain", version)
if tc.Present(version) {
t.Fatal("present before any install")
}
if err := tc.Ensure(context.Background(), version); err != nil {
t.Fatal(err)
}
if len(fake.calls) != 1 {
t.Fatalf("npm ran %d times, want once", len(fake.calls))
}
c := fake.calls[0]
joined := strings.Join(c.args, " ")
if c.name != "npm" || c.args[0] != "ci" || !strings.Contains(joined, "--ignore-scripts") || !strings.Contains(joined, "--no-bin-links") || !strings.Contains(joined, "--prefix "+dir) {
t.Errorf("ran %s %v, want npm ci --ignore-scripts --prefix %s", c.name, c.args, dir)
}
if lock, _ := os.ReadFile(filepath.Join(dir, "package-lock.json")); string(lock) != `{"lock":1}` {
t.Errorf("the directory holds lock %q, want the module's", lock)
}
if _, err := os.Stat(filepath.Join(dir, ".npmrc-install")); err == nil {
t.Errorf("the registry credential file was left in the toolchain directory")
}
if !tc.Present(version) {
t.Errorf("not present after the install")
}
if err := tc.Ensure(context.Background(), version); err != nil || len(fake.calls) != 1 {
t.Errorf("a present directory ran npm again: %d calls, %v", len(fake.calls), err)
}
}
// 1488/AC4: the install is told the mesh's registry for @novox with the account's credential, and only through a
// config file of its own, never on the command line.
func TestTheInstallNamesTheRegistryInAConfigFileNotOnTheCommandLine(t *testing.T) {
var npmrc string
fake := &fakeNpm{}
tc := toolchainIn(t, fake)
tc.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
for i, a := range args {
if a == "--userconfig" {
raw, _ := os.ReadFile(args[i+1])
npmrc = string(raw)
}
}
return fake.run(ctx, env, name, args...)
}
if err := tc.Ensure(context.Background(), "v1"); err != nil {
t.Fatal(err)
}
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n") ||
!strings.Contains(npmrc, "//git.novox.be/api/packages/novox/npm/:_auth=") {
t.Errorf("npmrc = %q", npmrc)
}
if strings.Contains(strings.Join(fake.calls[0].args, " "), "pw") {
t.Errorf("the credential is on the command line: %v", fake.calls[0].args)
}
}
// 1488/AC4: an install that fails leaves no directory taken for a toolchain, says why, and the next call tries again.
func TestAFailedInstallIsNotAToolchain(t *testing.T) {
fake := &fakeNpm{ran: forgekey.Ran{Status: 1, Stderr: "E401 unauthorized"}}
tc := toolchainIn(t, fake)
err := tc.Ensure(context.Background(), "v1")
if err == nil || !strings.Contains(err.Error(), "E401") {
t.Fatalf("err = %v, want the npm failure named", err)
}
if tc.Present("v1") {
t.Errorf("a failed install is present")
}
fake.ran = forgekey.Ran{}
if err := tc.Ensure(context.Background(), "v1"); err != nil || len(fake.calls) != 2 || !tc.Present("v1") {
t.Errorf("the retry: %v, %d calls", err, len(fake.calls))
}
}
// 1488/AC4: a module's steps run the install behind the guard: a refusing node installs nothing, an accepting one
// installs once however many steps follow.
func TestPrepareInstallsOnlyBehindAPassingGuard(t *testing.T) {
fake := &fakeNpm{}
tc := toolchainIn(t, fake)
states := newStates()
m := moduleWith(states, "v26.11.0")
m.Toolchain = &tc
if err := m.Prepare(context.Background()); err == nil || len(fake.calls) != 0 {
t.Fatalf("a refusing node: %v, %d npm calls", err, len(fake.calls))
}
m = moduleWith(states, "v26.10.0")
m.Toolchain = &tc
for i := 0; i < 3; i++ {
if err := m.Prepare(context.Background()); err != nil {
t.Fatal(err)
}
}
if len(fake.calls) != 1 {
t.Errorf("npm ran %d times for three steps, want once", len(fake.calls))
}
}
func TestRegistryFromTheBinding(t *testing.T) {
dir := t.TempDir()
b, s := filepath.Join(dir, "b.json"), filepath.Join(dir, "s")
os.WriteFile(s, []byte("secret\n"), 0o600)
os.WriteFile(b, []byte(`{"provision":"npm-package-registry","at":"10.42.0.1","as":"mesh-build-node-ace","serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`), 0o600)
r, err := ReadRegistry(b, s)
if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Username != "mesh-build-node-ace" || r.Password != "secret" {
t.Fatalf("= %+v, %v", r, err)
}
os.WriteFile(b, []byte(`{"at":"h","as":"a","serves":{"port":1,"npm-path":"/p/","public-root":"https://git.novox.be/p"}}`), 0o600)
if r, _ := ReadRegistry(b, s); r.URL != "https://git.novox.be/p/" {
t.Errorf("the public root is not preferred: %q", r.URL)
}
if _, err := ReadRegistry(filepath.Join(dir, "none"), s); err == nil {
t.Errorf("no binding was accepted")
}
}

Some files were not shown because too many files have changed in this diff Show More