diff --git a/.gitignore b/.gitignore index 77f4b90c..c61c7638 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image # binaries a local go build leaves in a module root /checks/paths/paths +/checks/environment/environment /modules/mesh-build-go/mesh-build-go /modules/mesh-delivery-gate/mesh-delivery-gate modules/mesh-config/cmd/mesh-config/mesh-config diff --git a/checks/environment/go.mod b/checks/environment/go.mod new file mode 100644 index 00000000..1ef96674 --- /dev/null +++ b/checks/environment/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/environment + +go 1.22 diff --git a/checks/environment/main.go b/checks/environment/main.go new file mode 100644 index 00000000..e8cd65a6 --- /dev/null +++ b/checks/environment/main.go @@ -0,0 +1,252 @@ +// Command environment refuses a module's code reading a MESH_* word of its environment that nothing gives it: not its +// manifest, and not the node-engine or the tool runner for every process (the lead's ruling on A3n, issue 1966). +// +// A MESH_* word is the mesh's contract with a module's process: the manifest sets it (an `env` of a build artifact or a +// resource, an `environment`'s `variables`, a `NAME=` line of a file the module ships for an env-file), or the mesh gives +// it to every process (Given). A word read and set by nothing is a contract nobody keeps: the program runs on its +// default or on nothing, and says so to nobody (MESH_REGISTRY_REACH answered every published reference without its +// registry; MESH_CONFIG_CALLERS held every window shut). Found here, it is refused: the code reads what the mesh gives +// (a manifest word, a config value through its tools, a seat's name), or the manifest sets it. +// +// A read is `os.Getenv` or `os.LookupEnv` of a literal or of a constant of the same directory (Go), and `env.MESH_*` or +// `env["MESH_*"]` (TypeScript, `process.env` included); tests, test directories, node_modules and dist are not read. The +// modules still reading such a word are listed in the file `unset` beside this check, ` ` a line, until +// each is fixed: a line no longer found is refused, so the list only shrinks. +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// Given is every MESH_* word the mesh gives a module's processes whatever their manifest says, and who gives it. +var Given = map[string]string{ + // The tool runner, to every bundle and to a module's own runtime (mesh-tools node-tools runtime.BundleEnv). + "MESH_MODULE": "the tool runner", + "MESH_SERVED_MODULE": "the tool runner", + "MESH_NODE": "the tool runner", + // The node-engine, to the tool runner it runs as the operator's account; its bundles inherit them (mesh-host + // compose/resolution runtime.go). + "MESH_OPERATOR_ACCOUNT": "the node-engine", + "MESH_OPERATOR_HOME": "the node-engine", + // The node-engine, to every module that yields to the node's builds (ADR 0553 ยง3; catalogue.BuildCPUShareWord). + "MESH_BUILD_CPU_SHARE": "the node-engine", +} + +const listFile = "unset" + +var ( + goRead = regexp.MustCompile(`os\.(?:Getenv|LookupEnv)\(\s*(?:"(MESH_[A-Z0-9_]+)"|([A-Za-z_][A-Za-z0-9_]*))\s*\)`) + goConst = regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\s*(?:string\s*)?=\s*"(MESH_[A-Z0-9_]+)"`) + tsRead = regexp.MustCompile(`\benv(?:\.|\[\s*["'])(MESH_[A-Z0-9_]+)`) + envLine = regexp.MustCompile(`(?m)^\s*(?:export\s+)?(MESH_[A-Z0-9_]+)=`) +) + +// skipped are the directories whose code no process of the module runs. +var skipped = map[string]bool{"node_modules": true, "dist": true, "test": true, "testdata": true, ".git": true} + +// Check answers, per module, each MESH_* word its code reads that its manifest does not set and the mesh does not +// give, with the files reading it. +func Check(root string) (map[string]map[string][]string, error) { + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + sort.Strings(manifests) + out := map[string]map[string][]string{} + for _, path := range manifests { + dir := filepath.Dir(path) + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var manifest any + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + set := map[string]bool{} + setBy(manifest, set) + reads := map[string]map[string]bool{} + consts := map[string]map[string]string{} // directory -> constant -> word + var goFiles []string + err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if p != dir && skipped[d.Name()] { + return filepath.SkipDir + } + return nil + } + name := d.Name() + text, err := os.ReadFile(p) + if err != nil { + return err + } + rel, _ := filepath.Rel(dir, p) + rel = filepath.ToSlash(rel) + switch { + case strings.HasSuffix(name, "_test.go"), strings.HasSuffix(name, ".test.ts"), strings.HasSuffix(name, ".d.ts"): + case strings.HasSuffix(name, ".go"): + goFiles = append(goFiles, p) + c := consts[filepath.Dir(p)] + if c == nil { + c = map[string]string{} + consts[filepath.Dir(p)] = c + } + for _, m := range goConst.FindAllStringSubmatch(string(text), -1) { + c[m[1]] = m[2] + } + case strings.HasSuffix(name, ".ts"): + for _, m := range tsRead.FindAllStringSubmatch(string(text), -1) { + read(reads, m[1], rel) + } + case name != "module.json": + // A file the module ships: a NAME= line is a word an env-file gives. + for _, m := range envLine.FindAllStringSubmatch(string(text), -1) { + set[m[1]] = true + } + } + return nil + }) + if err != nil { + return nil, err + } + for _, p := range goFiles { + text, _ := os.ReadFile(p) + rel, _ := filepath.Rel(dir, p) + for _, m := range goRead.FindAllStringSubmatch(string(text), -1) { + word := m[1] + if word == "" { + word = consts[filepath.Dir(p)][m[2]] + } + if word != "" { + read(reads, word, filepath.ToSlash(rel)) + } + } + } + module := filepath.Base(dir) + for word, files := range reads { + if set[word] || Given[word] != "" { + continue + } + if out[module] == nil { + out[module] = map[string][]string{} + } + for f := range files { + out[module][word] = append(out[module][word], f) + } + sort.Strings(out[module][word]) + } + } + return out, nil +} + +func read(reads map[string]map[string]bool, word, file string) { + if reads[word] == nil { + reads[word] = map[string]bool{} + } + reads[word][file] = true +} + +// setBy collects the words a manifest sets: every key of an `env` object and of an `environment`'s `variables`. +func setBy(v any, set map[string]bool) { + switch v := v.(type) { + case map[string]any: + for k, e := range v { + if m, ok := e.(map[string]any); ok && (k == "env" || k == "variables") { + for word := range m { + set[word] = true + } + } + setBy(e, set) + } + case []any: + for _, e := range v { + setBy(e, set) + } + } +} + +// Listed is the list's ` ` lines; a line starting with # and a blank line are not entries. +func Listed(path string) (map[string]bool, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + out := map[string]bool{} + s := bufio.NewScanner(f) + for s.Scan() { + line := strings.Join(strings.Fields(s.Text()), " ") + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out[line] = true + } + return out, s.Err() +} + +// Judge answers the reads not on the list (refused) and the list's lines no read stands for (stale). +func Judge(found map[string]map[string][]string, listed map[string]bool) (refused, stale []string) { + seen := map[string]bool{} + for module, words := range found { + for word := range words { + line := module + " " + word + seen[line] = true + if !listed[line] { + refused = append(refused, line) + } + } + } + for line := range listed { + if !seen[line] { + stale = append(stale, line) + } + } + sort.Strings(refused) + sort.Strings(stale) + return refused, stale +} + +func main() { + root := "." + if len(os.Args) > 1 { + root = os.Args[1] + } + found, err := Check(root) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + listed, err := Listed(listFile) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + refused, stale := Judge(found, listed) + for _, line := range refused { + module, word, _ := strings.Cut(line, " ") + fmt.Printf("%s reads %s (%s), which its manifest does not set and the mesh gives no process: set it in the "+ + "manifest, or read what the mesh gives\n", module, word, strings.Join(found[module][word], ", ")) + } + for _, line := range stale { + fmt.Printf("checks/environment/%s names %q, which no code reads unset any more: take it off the list\n", listFile, line) + } + if len(refused) > 0 || len(stale) > 0 { + os.Exit(1) + } + fmt.Printf("environment: every MESH_* word a module reads is set by its manifest or given by the mesh (%d still listed)\n", len(listed)) +} diff --git a/checks/environment/main_test.go b/checks/environment/main_test.go new file mode 100644 index 00000000..f79842cd --- /dev/null +++ b/checks/environment/main_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +// firstCount is how many lines the list held when it was first written: it only shrinks. +const firstCount = 63 + +func module(t *testing.T, root, name, manifest string, files map[string]string) { + t.Helper() + dir := filepath.Join(root, "modules", name) + for path, text := range map[string]string{"module.json": manifest} { + files[path] = text + } + for path, text := range files { + full := filepath.Join(dir, filepath.FromSlash(path)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } +} + +// A word read and set by nothing is found, in Go (a literal, a constant of the same directory) and in TypeScript; one +// the manifest sets (an env, an environment's variables, an env-file line it ships) or the mesh gives every process is +// not; tests, node_modules and dist are not read. +func TestAWordReadThatNothingSetsIsFound(t *testing.T) { + root := t.TempDir() + module(t, root, "reader", `{"module":"reader","build":{"artifacts":[{"name":"tools","env":{"MESH_SET":"x"}}]},`+ + `"resources":[{"id":"p","type":"process","environment":{"variables":{"MESH_VARIABLE":"y"}}}]}`, map[string]string{ + "cmd/a/main.go": `package main; const reach = "MESH_REACH"; var _ = os.Getenv(reach); var _ = os.Getenv("MESH_SET")`, + "cmd/a/other.go": `package main; var _, _ = os.LookupEnv("MESH_LITERAL"); var _ = os.Getenv("MESH_MODULE")`, + "cmd/a/write.go": `package main; var _ = []string{"MESH_WRITTEN_FOR_A_CHILD=1"}`, + "cmd/a/a_test.go": `package main; var _ = os.Getenv("MESH_IN_A_TEST")`, + "client.ts": `const a = process.env.MESH_TS ?? "d"; const b = env["MESH_BRACKET"]; const c = env.MESH_VARIABLE;`, + "files/env": "MESH_FROM_FILE=${config:x}\n", + "index.ts": `const f = env.MESH_FROM_FILE;`, + "test/x.test.ts": `env.MESH_IN_A_TEST_DIR`, + "node_modules/x.ts": `env.MESH_VENDORED`, + }) + module(t, root, "quiet", `{"module":"quiet"}`, map[string]string{"main.go": `package main; var _ = os.Getenv("MESH_NODE")`}) + got, err := Check(root) + if err != nil { + t.Fatal(err) + } + want := map[string]map[string][]string{"reader": { + "MESH_REACH": {"cmd/a/main.go"}, + "MESH_LITERAL": {"cmd/a/other.go"}, + "MESH_TS": {"client.ts"}, + "MESH_BRACKET": {"client.ts"}, + }} + if !reflect.DeepEqual(got, want) { + t.Fatalf("found %v, want %v", got, want) + } + refused, stale := Judge(got, map[string]bool{"reader MESH_REACH": true, "reader MESH_GONE": true}) + if !reflect.DeepEqual(refused, []string{"reader MESH_BRACKET", "reader MESH_LITERAL", "reader MESH_TS"}) || + !reflect.DeepEqual(stale, []string{"reader MESH_GONE"}) { + t.Errorf("refused %v, stale %v", refused, stale) + } +} + +// The catalogue itself: every word a module reads is set or given, but the list's, and the list only shrinks. +func TestTheCatalogueReadsOnlyWordsTheMeshSets(t *testing.T) { + found, err := Check(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + listed, err := Listed(listFile) + if err != nil { + t.Fatal(err) + } + refused, stale := Judge(found, listed) + for _, line := range refused { + t.Errorf("%s: read, set by nothing, and not on checks/environment/unset", line) + } + for _, line := range stale { + t.Errorf("checks/environment/unset names %q, which no code reads unset any more", line) + } + if len(listed) > firstCount { + t.Errorf("checks/environment/unset holds %d lines, more than its first %d: it only shrinks", len(listed), firstCount) + } +} diff --git a/checks/environment/reads-outside b/checks/environment/reads-outside new file mode 100644 index 00000000..e1553863 --- /dev/null +++ b/checks/environment/reads-outside @@ -0,0 +1,5 @@ +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and the code and files of its module. +modules diff --git a/checks/environment/unset b/checks/environment/unset new file mode 100644 index 00000000..5840f12b --- /dev/null +++ b/checks/environment/unset @@ -0,0 +1,67 @@ +# The MESH_* words a module's code still reads that its manifest does not set and the mesh gives no process, +# ` ` a line, as checks/environment first found them (issue 1966). Each is a contract nobody keeps: fix +# it (the manifest sets it, or the code reads what the mesh gives) and take its line off. A line no code reads unset any +# more is refused, so this list only shrinks; a new read set by nothing is refused, never added here. +baserow MESH_BASEROW_EMAIL +baserow MESH_BASEROW_HOST +baserow MESH_BASEROW_PASSWORD +claude-code MESH_AGENT_ACCOUNT +claude-code MESH_AGENT_HOME +claude-licence-manager MESH_ANTHROPIC_TOKEN_ENDPOINT +claude-licence-manager MESH_ANTHROPIC_USAGE_ENDPOINT +confluence MESH_CONFLUENCE_EMAIL +confluence MESH_CONFLUENCE_TOKEN +confluence MESH_CONFLUENCE_URL +disk-load MESH_DISK_MEASURE_EVERY +gitlab MESH_GITLAB_TOKEN +gitlab MESH_GITLAB_URL +grafana MESH_GRAFANA_PASSWORD +grafana MESH_GRAFANA_TOKEN +grafana MESH_GRAFANA_USER +home-assistant MESH_HOMEASSISTANT_TOKEN +home-assistant MESH_HOMEASSISTANT_WAIT_SECONDS +home-assistant MESH_HOMEASSISTANT_WATCH +icecast MESH_ICECAST_ADMIN_PASSWORD +icecast MESH_ICECAST_ADMIN_USER +influxdb MESH_INFLUXDB_ORG +influxdb MESH_INFLUXDB_TOKEN +jira MESH_JIRA_EMAIL +jira MESH_JIRA_TOKEN +jira MESH_JIRA_URL +letta MESH_LETTA_PASSWORD +mailu MESH_MAILU_API_KEY +mailu MESH_MAILU_DOMAIN +mesh-cli MESH_CONSOLE +mesh-cli MESH_VERB +mesh-files MESH_OWNED_PATHS +mesh-files MESH_SECRET_PLACES +mesh-reads MESH_SECRET_PLACES +minio MESH_MINIO_ROOT_PASSWORD +mongodb MESH_MONGODB_AUTHSOURCE +mongodb MESH_MONGODB_HOST +mongodb MESH_MONGODB_PASSWORD +mongodb MESH_MONGODB_PORT +mongodb MESH_MONGODB_USER +mosquitto MESH_MQTT_ADMIN_USER +mosquitto MESH_MQTT_BROKER_GID +mosquitto MESH_MQTT_BROKER_UID +mosquitto MESH_MQTT_HOST +mosquitto MESH_MQTT_PASSWORD +mosquitto MESH_MQTT_PORT +mssql MESH_MSSQL_HOST +mssql MESH_MSSQL_PASSWORD +mssql MESH_MSSQL_PORT +mssql MESH_MSSQL_READER_PASSWORD +mssql MESH_MSSQL_USER +nextcloud MESH_NEXTCLOUD_ADMIN_PASSWORD +nextcloud MESH_NEXTCLOUD_CONTAINER +nodered MESH_NODERED_TOKEN +nodered MESH_NODERED_WAIT_SECONDS +redis MESH_REDIS_HOST +redis MESH_REDIS_PASSWORD +redis MESH_REDIS_PORT +redis MESH_REDIS_USERNAME +restic MESH_BACKUP_HOUR +unifi MESH_UNIFI_PASSWORD +unifi MESH_UNIFI_SITE +unifi MESH_UNIFI_USERNAME