diff --git a/modules/docker-compose/README.md b/modules/docker-compose/README.md new file mode 100644 index 0000000..e712833 --- /dev/null +++ b/modules/docker-compose/README.md @@ -0,0 +1,65 @@ +# docker-compose + +Compose, for development work on the two workstations (novox/hq research 027/02: "the distribution's +package and nothing else", assigned only to the workstations; to-be 42 phase 2 step 9). The servers +run nothing through compose. + +## Owns + +| what | where | +|---|---| +| compose, as the container runtime's plugin (`docker compose`) and as `docker-compose` | package `docker-compose` | + +Nothing else. The runtime, its configuration, buildx and the `docker` group are the `docker` module's +(to-be 42 phase 1 step 8). This module needs that runtime on the machine. Until the `docker` module +holds `node-container-runtime` there, nothing in the mesh says so, and the tools answer that the +runtime is missing or unreachable rather than an empty list. + +## Improves + +- **An owner for a package both workstations already carry.** On both, `docker-compose` 5.5.0 is + installed explicitly by hand, as the plugin in `/usr/lib/docker/cli-plugins`. Assigning the module + changes nothing on disk; from then on the package is the mesh's, upgraded with the machine and + given back when the module goes. +- **The projects become visible from the mesh** without a shell on the machine: which are running, + where their files are, their containers, logs and rendered configuration. +- **No account-level copy of the plugin** exists on either workstation (`~/.docker/cli-plugins` is + empty), so there is no second compose to remove. + +## Tools + +All answer JSON; `(r)` reads, `(a)` acts. They run as the operator account, which reaches the runtime +through the `docker` group. Nothing goes through `sudo`. + +A project is named by `dir`, its absolute directory, or by `project`, its name. A directory docker +already knows a project for is that project, with the files it was started from, overrides included. +Any other directory must hold a compose file. + +| tool | what | +|---|---| +| `docker_compose_projects` (r) | every project docker knows, running or stopped: status, working directory (from the containers' labels), compose files, services, containers running of total | +| `docker_compose_ps` (r) | one project's containers: service, state, health, exit code, image, published ports | +| `docker_compose_logs` (r) | the last lines per service (default 200, at most 5000), optionally `since`; cut at 256 KiB | +| `docker_compose_config` (r) | the rendered configuration. Values of environment variables, build arguments and labels whose names suggest a secret, and inline secret or config content, are replaced with `[redacted]`, and the answer counts them | +| `docker_compose_up` (a) | `up --detach`, with the pull policy (default `missing`) and optionally `--build`, for all or some services | +| `docker_compose_down` (a) | `down`: containers and networks. **Volumes are kept**: no tool here removes data | +| `docker_compose_restart` (a) | restart all or some services | +| `docker_compose_pull` (a) | pull images without starting anything | +| `docker_compose_job` (r) | a long act's state: running or finished, exit status, the end of its output; without an id, every act this process knows | + +**Acts are jobs.** An `up` that pulls or builds takes minutes, and the runtime gives a call 30 s. Each +act runs inside the tool's process for up to 15 minutes, and is waited on for 18 s. A finished act is +answered with its output, and a failed one as an error. One still running is answered with its job id, +which `docker_compose_job` follows. A job ends if the runtime restarts the bundle. + +## Leaves as found + +The projects themselves are the operator's work, under the operator's directories. Measured on +2026-10-04: + +- **laptop:** `anton-lavinmq` and `anton-traefik` running, `anton-redis` stopped. +- **desktop:** `anton-lavinmq`, `lavinmq` (from `/services/lavinmq`, a predecessor's directory) and + `registry` running. + +Whether the desktop's `lavinmq` and `registry` should still run is the operator's call; +`docker_compose_down` with their directory stops them. diff --git a/modules/docker-compose/cmd/docker-compose-tools/compose.go b/modules/docker-compose/cmd/docker-compose-tools/compose.go new file mode 100644 index 0000000..566b367 --- /dev/null +++ b/modules/docker-compose/cmd/docker-compose-tools/compose.go @@ -0,0 +1,455 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// Project is one compose project as docker knows it. +type Project struct { + Name string `json:"name"` + Status string `json:"status,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + ConfigFiles []string `json:"config_files"` + Services []string `json:"services"` + Running int `json:"running"` + Containers int `json:"containers"` +} + +// ProjectsAnswer is what docker_compose_projects answers. +type ProjectsAnswer struct { + Count int `json:"count"` + Projects []Project `json:"projects"` +} + +// composeFiles are the names compose looks for in a directory, in its order. +var composeFiles = []string{"compose.yaml", "compose.yml", "docker-compose.yaml", "docker-compose.yml"} + +// statDir says whether a path is a directory. Tests replace it. +var statDir = func(p string) bool { + info, err := os.Stat(p) + return err == nil && info.IsDir() +} + +// statFile says whether a path is a regular file. Tests replace it. +var statFile = func(p string) bool { + info, err := os.Stat(p) + return err == nil && info.Mode().IsRegular() +} + +// docker runs one docker command and names a daemon the account cannot reach as such. +func docker(args ...string) (Result, error) { + r, err := call(Cmd{Name: "docker", Args: args}) + if err != nil && strings.Contains(r.Stderr, "permission denied") && strings.Contains(r.Stderr, "docker.sock") { + return r, fmt.Errorf("the account cannot reach the container runtime's socket (permission denied): it is not in the docker group, or has not logged in since it was added. The docker module owns the group's members") + } + if err != nil && strings.Contains(r.Stderr, "Cannot connect to the Docker daemon") { + return r, fmt.Errorf("the container runtime is not running on this machine: %s", firstLine(r.Stderr)) + } + return r, err +} + +// Projects merges what compose lists with what the containers' labels say. +func Projects() (ProjectsAnswer, error) { + r, err := docker("compose", "ls", "--all", "--format", "json") + if err != nil { + return ProjectsAnswer{}, err + } + var listed []struct { + Name string `json:"Name"` + Status string `json:"Status"` + ConfigFiles string `json:"ConfigFiles"` + } + if s := strings.TrimSpace(r.Stdout); s != "" { + if err := json.Unmarshal([]byte(s), &listed); err != nil { + return ProjectsAnswer{}, fmt.Errorf("docker compose ls answered what is not JSON: %v", err) + } + } + by := map[string]*Project{} + get := func(name string) *Project { + if by[name] == nil { + by[name] = &Project{Name: name, ConfigFiles: []string{}, Services: []string{}} + } + return by[name] + } + for _, l := range listed { + p := get(l.Name) + p.Status = l.Status + p.ConfigFiles = splitFiles(l.ConfigFiles) + } + r, err = docker("ps", "-a", "--filter", "label=com.docker.compose.project", "--format", + `{{.Label "com.docker.compose.project"}}`+"\t"+`{{.Label "com.docker.compose.project.working_dir"}}`+"\t"+ + `{{.Label "com.docker.compose.project.config_files"}}`+"\t"+`{{.Label "com.docker.compose.service"}}`+"\t{{.State}}") + if err != nil { + return ProjectsAnswer{}, err + } + services := map[string]map[string]bool{} + for _, l := range lines(r.Stdout) { + f := strings.Split(l, "\t") + if len(f) < 5 || f[0] == "" { + continue + } + p := get(f[0]) + if p.WorkingDir == "" { + p.WorkingDir = f[1] + } + if len(p.ConfigFiles) == 0 { + p.ConfigFiles = splitFiles(f[2]) + } + if services[f[0]] == nil { + services[f[0]] = map[string]bool{} + } + if f[3] != "" { + services[f[0]][f[3]] = true + } + p.Containers++ + if f[4] == "running" { + p.Running++ + } + } + out := ProjectsAnswer{Projects: []Project{}} + for name, p := range by { + for s := range services[name] { + p.Services = append(p.Services, s) + } + sort.Strings(p.Services) + if p.WorkingDir == "" && len(p.ConfigFiles) > 0 { + p.WorkingDir = filepath.Dir(p.ConfigFiles[0]) + } + out.Projects = append(out.Projects, *p) + } + sort.Slice(out.Projects, func(i, k int) bool { return out.Projects[i].Name < out.Projects[k].Name }) + out.Count = len(out.Projects) + return out, nil +} + +func splitFiles(s string) []string { + out := []string{} + for _, f := range strings.Split(s, ",") { + if f = strings.TrimSpace(f); f != "" { + out = append(out, f) + } + } + return out +} + +// Target is the project a tool acts on, and how compose is told which it is. +type Target struct { + Project string `json:"project,omitempty"` + Dir string `json:"dir,omitempty"` + Files []string `json:"files,omitempty"` +} + +// args are compose's own options naming the target. +func (t Target) args() []string { + out := []string{"compose"} + if t.Dir != "" { + out = append(out, "--project-directory", t.Dir) + } + for _, f := range t.Files { + out = append(out, "-f", f) + } + if t.Project != "" { + out = append(out, "-p", t.Project) + } + return out +} + +var projectName = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) + +// targetOf reads dir or project. A directory docker already knows a project for is that project, +// with the files it was started from; otherwise it must hold a compose file. needFiles says the +// tool reads the files (config, up, pull), so a project known only by its containers is not enough. +func targetOf(args map[string]any, needFiles bool) (Target, error) { + dir, err := optText(args, "dir", "") + if err != nil { + return Target{}, err + } + name, err := optText(args, "project", "") + if err != nil { + return Target{}, err + } + if dir == "" && name == "" { + return Target{}, fmt.Errorf("give dir, the project's directory, or project, its name") + } + if dir != "" { + if !filepath.IsAbs(dir) { + return Target{}, fmt.Errorf("dir must be an absolute path, not %q", dir) + } + dir = filepath.Clean(dir) + if !statDir(dir) { + return Target{}, fmt.Errorf("%s is not a directory on this machine", dir) + } + } + if name != "" && !projectName.MatchString(name) { + return Target{}, fmt.Errorf("%q is not a compose project name", name) + } + known, err := Projects() + if err != nil { + return Target{}, err + } + for _, p := range known.Projects { + if (dir != "" && p.WorkingDir == dir) || (dir == "" && p.Name == name) { + if name != "" && p.Name != name { + continue + } + t := Target{Project: p.Name, Dir: p.WorkingDir} + present := len(p.ConfigFiles) > 0 + for _, f := range p.ConfigFiles { + present = present && statFile(f) + } + if present { + t.Files = p.ConfigFiles + } else if needFiles && !hasComposeFile(t.Dir) { + return Target{}, fmt.Errorf("project %s was started from %s, which is no longer there", p.Name, strings.Join(p.ConfigFiles, ", ")) + } + return t, nil + } + } + if dir == "" { + return Target{}, fmt.Errorf("no compose project named %s is known to docker here: give dir, its directory", name) + } + if !hasComposeFile(dir) { + return Target{}, fmt.Errorf("%s holds no compose file (%s)", dir, strings.Join(composeFiles, ", ")) + } + return Target{Dir: dir, Project: name}, nil +} + +func hasComposeFile(dir string) bool { + for _, f := range composeFiles { + if statFile(filepath.Join(dir, f)) { + return true + } + } + return false +} + +// Container is one of a project's containers. +type Container struct { + Name string `json:"name"` + Service string `json:"service"` + State string `json:"state"` + Status string `json:"status"` + Health string `json:"health,omitempty"` + ExitCode int `json:"exit_code"` + Image string `json:"image"` + Ports []string `json:"ports"` +} + +// PsAnswer is what docker_compose_ps answers. +type PsAnswer struct { + Target Target `json:"target"` + Containers []Container `json:"containers"` +} + +// jsonObjects reads compose's JSON output, which is one array or one object per line by version. +func jsonObjects(s string, into any) error { + s = strings.TrimSpace(s) + if s == "" { + s = "[]" + } + if !strings.HasPrefix(s, "[") { + s = "[" + strings.Join(lines(s), ",") + "]" + } + return json.Unmarshal([]byte(s), into) +} + +// Ps answers a project's containers. +func Ps(t Target) (PsAnswer, error) { + r, err := docker(append(t.args(), "ps", "-a", "--format", "json")...) + if err != nil { + return PsAnswer{}, err + } + var raw []struct { + Name string `json:"Name"` + Service string `json:"Service"` + State string `json:"State"` + Status string `json:"Status"` + Health string `json:"Health"` + ExitCode int `json:"ExitCode"` + Image string `json:"Image"` + Publishers []struct { + URL string `json:"URL"` + TargetPort int `json:"TargetPort"` + PublishedPort int `json:"PublishedPort"` + Protocol string `json:"Protocol"` + } `json:"Publishers"` + } + if err := jsonObjects(r.Stdout, &raw); err != nil { + return PsAnswer{}, fmt.Errorf("docker compose ps answered what is not JSON: %v", err) + } + out := PsAnswer{Target: t, Containers: []Container{}} + for _, c := range raw { + ports := []string{} + for _, p := range c.Publishers { + if p.PublishedPort == 0 { + continue + } + ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", p.URL, p.PublishedPort, p.TargetPort, p.Protocol)) + } + out.Containers = append(out.Containers, Container{Name: c.Name, Service: c.Service, State: c.State, Status: c.Status, + Health: c.Health, ExitCode: c.ExitCode, Image: c.Image, Ports: ports}) + } + return out, nil +} + +// LogsAnswer is what docker_compose_logs answers. +type LogsAnswer struct { + Target Target `json:"target"` + Lines []string `json:"lines"` + Truncated bool `json:"truncated,omitempty"` +} + +var since = regexp.MustCompile(`^[0-9A-Za-z:.+-]+$`) + +// Logs answers a project's last lines. +func Logs(t Target, services []string, n int, from string) (LogsAnswer, error) { + args := append(t.args(), "logs", "--no-color", "--timestamps", "--tail", fmt.Sprint(n)) + if from != "" { + if !since.MatchString(from) { + return LogsAnswer{}, fmt.Errorf("since %q is neither a duration nor a timestamp", from) + } + args = append(args, "--since", from) + } + for _, s := range services { + if err := plainName("service", s); err != nil { + return LogsAnswer{}, err + } + } + r, err := docker(append(args, services...)...) + if err != nil { + return LogsAnswer{}, err + } + // compose writes the containers' output on its stdout, and its own complaints on stderr. + return LogsAnswer{Target: t, Lines: lines(r.Stdout), Truncated: r.Truncated}, nil +} + +// ConfigAnswer is what docker_compose_config answers. +type ConfigAnswer struct { + Target Target `json:"target"` + Services []string `json:"services"` + Redacted int `json:"redacted"` + Rendered map[string]any `json:"rendered"` +} + +// secretish is a name whose value is not shown. +var secretish = regexp.MustCompile(`(?i)(pass|secret|token|key|credential|auth|private|cert|cookie|session|salt|dsn|api)`) + +// redact replaces the values of secret-looking names in the maps compose renders. +func redact(v any, count *int) { + switch x := v.(type) { + case map[string]any: + for k, child := range x { + switch k { + case "environment", "args", "labels", "build_args": + if m, ok := child.(map[string]any); ok { + for name, val := range m { + if val != nil && secretish.MatchString(name) { + m[name] = "[redacted]" + *count++ + } + } + continue + } + case "content": + // An inline config or secret: its content is the secret itself. + if _, ok := child.(string); ok { + x[k] = "[redacted]" + *count++ + continue + } + } + redact(child, count) + } + case []any: + for _, child := range x { + redact(child, count) + } + } +} + +// Config answers the rendered configuration. +func Config(t Target) (ConfigAnswer, error) { + r, err := docker(append(t.args(), "config", "--format", "json")...) + if err != nil { + return ConfigAnswer{}, err + } + var rendered map[string]any + if err := json.Unmarshal([]byte(r.Stdout), &rendered); err != nil { + return ConfigAnswer{}, fmt.Errorf("docker compose config answered what is not JSON: %v", err) + } + out := ConfigAnswer{Target: t, Services: []string{}, Rendered: rendered} + if s, ok := rendered["services"].(map[string]any); ok { + for name := range s { + out.Services = append(out.Services, name) + } + sort.Strings(out.Services) + } + redact(rendered, &out.Redacted) + return out, nil +} + +// ActAnswer is what an act answers: the target and the job that carries it. +type ActAnswer struct { + Act string `json:"act"` + Target Target `json:"target"` + Job Job `json:"job"` +} + +func act(name string, t Target, extra ...string) (ActAnswer, error) { + j, err := actAsJob(Cmd{Name: "docker", Args: append(append(t.args(), name), extra...)}) + if err != nil { + return ActAnswer{}, err + } + return ActAnswer{Act: name, Target: t, Job: j}, nil +} + +func checkServices(services []string) error { + for _, s := range services { + if err := plainName("service", s); err != nil { + return err + } + } + return nil +} + +// Up brings a project up, detached. +func Up(t Target, services []string, build bool, pull string) (ActAnswer, error) { + if err := oneOf("pull", pull, "missing", "always", "never"); err != nil { + return ActAnswer{}, err + } + if err := checkServices(services); err != nil { + return ActAnswer{}, err + } + extra := []string{"--detach", "--pull", pull} + if build { + extra = append(extra, "--build") + } + return act("up", t, append(extra, services...)...) +} + +// Down stops and removes a project's containers and networks, keeping its volumes. +func Down(t Target) (ActAnswer, error) { + return act("down", t) +} + +// Restart restarts a project's containers. +func Restart(t Target, services []string) (ActAnswer, error) { + if err := checkServices(services); err != nil { + return ActAnswer{}, err + } + return act("restart", t, services...) +} + +// Pull pulls a project's images. +func Pull(t Target, services []string) (ActAnswer, error) { + if err := checkServices(services); err != nil { + return ActAnswer{}, err + } + return act("pull", t, services...) +} diff --git a/modules/docker-compose/cmd/docker-compose-tools/compose_test.go b/modules/docker-compose/cmd/docker-compose-tools/compose_test.go new file mode 100644 index 0000000..750939a --- /dev/null +++ b/modules/docker-compose/cmd/docker-compose-tools/compose_test.go @@ -0,0 +1,222 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestTheManifestIsComposesPackageAndNothingElse(t *testing.T) { + m := readManifest(t) + holdsTheBundle(t, m, "docker_compose") + if got := strings.Join(m.packages(), ","); got != "docker-compose" { + t.Errorf("packages %s: buildx and the runtime are the docker module's", got) + } + if len(m.Resources) != 1 { + t.Errorf("one resource, the package: %v", m.Resources) + } +} + +const lsJSON = `[{"Name":"anton-lavinmq","Status":"running(1)","ConfigFiles":"/home/op/hub/lavinmq/docker-compose.yml"},{"Name":"old","Status":"exited(2)","ConfigFiles":"/srv/old/compose.yaml,/srv/old/compose.override.yaml"}]` + +const psLabels = "anton-lavinmq\t/home/op/hub/lavinmq\t/home/op/hub/lavinmq/docker-compose.yml\tlavinmq\trunning\n" + + "old\t/srv/old\t/srv/old/compose.yaml,/srv/old/compose.override.yaml\tweb\texited\n" + + "old\t/srv/old\t/srv/old/compose.yaml,/srv/old/compose.override.yaml\tdb\texited\n" + +// aDocker answers compose ls and the labelled ps, and hands every other line to rest. +func aDocker(t *testing.T, rest func(line string) Result) *fake { + return using(t, func(line string, c Cmd) Result { + switch { + case strings.HasPrefix(line, "docker compose ls"): + return ok(lsJSON) + case strings.HasPrefix(line, "docker ps -a --filter label=com.docker.compose.project"): + return ok(psLabels) + } + if rest != nil { + return rest(line) + } + return ok("") + }) +} + +func onDisk(t *testing.T, dirs, files []string) { + t.Helper() + wasD, wasF := statDir, statFile + in := func(set []string) func(string) bool { + return func(p string) bool { + for _, s := range set { + if s == p { + return true + } + } + return false + } + } + statDir, statFile = in(dirs), in(files) + t.Cleanup(func() { statDir, statFile = wasD, wasF }) +} + +func TestProjectsMergesComposesListWithTheContainersLabels(t *testing.T) { + aDocker(t, nil) + got, err := Projects() + if err != nil || got.Count != 2 { + t.Fatalf("%+v %v", got, err) + } + p := got.Projects[0] + if p.Name != "anton-lavinmq" || p.WorkingDir != "/home/op/hub/lavinmq" || p.Running != 1 || p.Containers != 1 || p.Status != "running(1)" { + t.Errorf("%+v", p) + } + o := got.Projects[1] + if strings.Join(o.Services, ",") != "db,web" || len(o.ConfigFiles) != 2 || o.Running != 0 || o.Containers != 2 { + t.Errorf("%+v", o) + } +} + +func TestProjectsNamesADaemonTheAccountCannotReach(t *testing.T) { + using(t, func(string, Cmd) Result { + return Result{Status: 1, Stderr: "permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock"} + }) + if _, err := Projects(); err == nil || !strings.Contains(err.Error(), "docker group") { + t.Fatalf("%v", err) + } +} + +func TestATargetIsAKnownProjectWithItsFilesOrADirectoryHoldingAComposeFile(t *testing.T) { + aDocker(t, nil) + onDisk(t, []string{"/home/op/hub/lavinmq", "/srv/new", "/srv/empty", "/srv/old"}, + []string{"/home/op/hub/lavinmq/docker-compose.yml", "/srv/new/compose.yaml"}) + got, err := targetOf(map[string]any{"dir": "/home/op/hub/lavinmq/"}, true) + if err != nil || got.Project != "anton-lavinmq" || len(got.Files) != 1 { + t.Fatalf("a known directory: %+v %v", got, err) + } + if a := strings.Join(got.args(), " "); a != "compose --project-directory /home/op/hub/lavinmq -f /home/op/hub/lavinmq/docker-compose.yml -p anton-lavinmq" { + t.Errorf("args %s", a) + } + got, err = targetOf(map[string]any{"dir": "/srv/new"}, true) + if err != nil || got.Project != "" || got.Dir != "/srv/new" { + t.Fatalf("a new directory: %+v %v", got, err) + } + got, err = targetOf(map[string]any{"project": "old"}, false) + if err != nil || got.Dir != "/srv/old" || got.Files != nil { + t.Fatalf("a project whose files are gone, for an act that needs none: %+v %v", got, err) + } + if _, err := targetOf(map[string]any{"project": "old"}, true); err == nil || !strings.Contains(err.Error(), "no longer there") { + t.Errorf("its files are needed: %v", err) + } + for _, bad := range []map[string]any{{}, {"dir": "rel/path"}, {"dir": "/nope"}, {"dir": "/srv/empty"}, {"project": "Bad Name"}, {"project": "unknown"}} { + if _, err := targetOf(bad, false); err == nil { + t.Errorf("%v was accepted", bad) + } + } +} + +func TestPsReadsEitherJSONShapeAndKeepsOnlyPublishedPorts(t *testing.T) { + aDocker(t, func(line string) Result { + return ok(`{"Name":"a-web-1","Service":"web","State":"running","Status":"Up 2 hours","Health":"healthy","ExitCode":0,"Image":"nginx","Publishers":[{"URL":"0.0.0.0","TargetPort":80,"PublishedPort":8080,"Protocol":"tcp"},{"URL":"","TargetPort":443,"PublishedPort":0,"Protocol":"tcp"}]} +{"Name":"a-db-1","Service":"db","State":"exited","Status":"Exited (1)","ExitCode":1,"Image":"postgres","Publishers":[]}`) + }) + got, err := Ps(Target{Project: "a"}) + if err != nil || len(got.Containers) != 2 { + t.Fatalf("%+v %v", got, err) + } + if c := got.Containers[0]; strings.Join(c.Ports, ",") != "0.0.0.0:8080->80/tcp" || c.Health != "healthy" { + t.Errorf("%+v", c) + } + var arr []map[string]any + if err := jsonObjects(`[{"Name":"x"}]`, &arr); err != nil || len(arr) != 1 { + t.Errorf("an array: %v %v", arr, err) + } +} + +func TestLogsAreBoundedAndRefuseAnOptionInAName(t *testing.T) { + f := aDocker(t, func(line string) Result { return Result{Stdout: "web-1 | a\nweb-1 | b\n", Truncated: true} }) + got, err := Logs(Target{Project: "a"}, []string{"web"}, 50, "10m") + if err != nil || len(got.Lines) != 2 || !got.Truncated { + t.Fatalf("%+v %v", got, err) + } + if l := f.lines()[0]; l != "docker compose -p a logs --no-color --timestamps --tail 50 --since 10m web" { + t.Errorf("%s", l) + } + if _, err := Logs(Target{Project: "a"}, []string{"--follow"}, 5, ""); err == nil { + t.Error("an option as a service") + } + if _, err := Logs(Target{Project: "a"}, nil, 5, "1h; rm"); err == nil { + t.Error("a since that is neither") + } +} + +func TestConfigRedactsSecretLookingValuesAndInlineContent(t *testing.T) { + aDocker(t, func(string) Result { + return ok(`{"name":"a","services":{"web":{"image":"nginx","environment":{"DB_PASSWORD":"hunter2","PORT":"80","API_TOKEN":"t"},"build":{"args":{"NPM_TOKEN":"n","NODE_ENV":"production"}}}},"secrets":{"s":{"content":"raw"}}}`) + }) + got, err := Config(Target{Dir: "/srv/a"}) + if err != nil || got.Redacted != 4 || strings.Join(got.Services, ",") != "web" { + t.Fatalf("%+v %v", got, err) + } + raw, _ := json.Marshal(got.Rendered) + for _, secret := range []string{"hunter2", `"t"`, `"n"`, "raw"} { + if strings.Contains(string(raw), secret) { + t.Errorf("%s shown: %s", secret, raw) + } + } + for _, kept := range []string{`"PORT":"80"`, `"NODE_ENV":"production"`, `"image":"nginx"`} { + if !strings.Contains(string(raw), kept) { + t.Errorf("%s hidden: %s", kept, raw) + } + } +} + +func TestConfigAnInvalidFileIsComposesError(t *testing.T) { + aDocker(t, func(string) Result { + return Result{Status: 15, Stderr: "services.web Additional property foo is not allowed"} + }) + if _, err := Config(Target{Dir: "/srv/a"}); err == nil || !strings.Contains(err.Error(), "Additional property") { + t.Fatalf("%v", err) + } +} + +func TestActsAreJobsAsTheAccountKeepVolumesAndCheckTheirArguments(t *testing.T) { + f := aDocker(t, nil) + tg := Target{Dir: "/srv/a", Project: "a"} + if got, err := Up(tg, []string{"web"}, true, "always"); err != nil || got.Job.Running || got.Act != "up" { + t.Fatalf("%+v %v", got, err) + } + if _, err := Down(tg); err != nil { + t.Fatal(err) + } + if _, err := Restart(tg, nil); err != nil { + t.Fatal(err) + } + if _, err := Pull(tg, nil); err != nil { + t.Fatal(err) + } + want := []string{ + "docker compose --project-directory /srv/a -p a up --detach --pull always --build web", + "docker compose --project-directory /srv/a -p a down", + "docker compose --project-directory /srv/a -p a restart", + "docker compose --project-directory /srv/a -p a pull", + } + if got := strings.Join(f.lines(), "\n"); got != strings.Join(want, "\n") { + t.Errorf("asked\n%s\nwant\n%s", got, strings.Join(want, "\n")) + } + for _, l := range f.lines() { + if strings.Contains(l, "sudo") || strings.Contains(l, "-v") || strings.Contains(l, "--volumes") { + t.Errorf("%s", l) + } + } + if _, err := Up(tg, nil, false, "sometimes"); err == nil { + t.Error("an unknown pull policy") + } + if _, err := Restart(tg, []string{"-t"}); err == nil { + t.Error("an option as a service") + } +} + +func TestAFailedActIsAnError(t *testing.T) { + aDocker(t, func(string) Result { + return Result{Status: 1, Stderr: "Error response from daemon: port is already allocated"} + }) + if _, err := Up(Target{Dir: "/srv/a"}, nil, false, "missing"); err == nil || !strings.Contains(err.Error(), "already allocated") { + t.Fatalf("%v", err) + } +} diff --git a/modules/docker-compose/cmd/docker-compose-tools/jobs.go b/modules/docker-compose/cmd/docker-compose-tools/jobs.go new file mode 100644 index 0000000..3885479 --- /dev/null +++ b/modules/docker-compose/cmd/docker-compose-tools/jobs.go @@ -0,0 +1,151 @@ +package main + +// jobs.go is the same file in the bundles whose acts can outlast one call (flatpak, docker-compose): +// an install or an `up` that pulls images takes minutes, and the runtime gives a call 30 s. Such an +// act is started as a job inside this process, waited on for a while, and answered either finished +// or with the job's id for the module's `_job` tool to follow. A job ends with this process: if the +// runtime restarts the bundle, a running job is cut off, and its id is then unknown. + +import ( + "fmt" + "sort" + "strings" + "sync" + "time" +) + +// JobLimit is the longest a job may run; JobWait how long an act waits before answering a job id. +const ( + JobLimit = 15 * time.Minute + JobWait = 18 * time.Second + keptJobs = 50 +) + +// Job is one long act, as its tool answers it. +type Job struct { + ID string `json:"job"` + Command string `json:"command"` + Started time.Time `json:"started"` + Finished *time.Time `json:"finished,omitempty"` + Running bool `json:"running"` + Status *int `json:"status,omitempty"` + Error string `json:"error,omitempty"` + Output string `json:"output,omitempty"` + Truncated bool `json:"truncated,omitempty"` + done chan struct{} +} + +type jobBook struct { + mu sync.Mutex + seq int + jobs map[string]*Job +} + +var jobs = &jobBook{jobs: map[string]*Job{}} + +// startJob runs c in the background, held to JobLimit. +func startJob(c Cmd) *Job { + c.Timeout = JobLimit + name, args := argv(c) + jobs.mu.Lock() + jobs.seq++ + j := &Job{ID: fmt.Sprintf("%d-%d", time.Now().Unix(), jobs.seq), Command: strings.TrimSpace(name + " " + strings.Join(args, " ")), + Started: time.Now().UTC(), Running: true, done: make(chan struct{})} + jobs.jobs[j.ID] = j + jobs.forgetOldest() + jobs.mu.Unlock() + go func() { + r := run(c) + var err error + if r.Status != 0 || r.Error != "" { + err = failure(c, r) + } + jobs.mu.Lock() + now := time.Now().UTC() + j.Finished, j.Running = &now, false + status := r.Status + j.Status = &status + if err != nil { + j.Error = err.Error() + } + j.Output = tail(strings.TrimSpace(r.Stdout+"\n"+r.Stderr), 16<<10) + j.Truncated = r.Truncated || len(r.Stdout)+len(r.Stderr) > 16<<10 + jobs.mu.Unlock() + close(j.done) + }() + return j +} + +// forgetOldest keeps the book bounded; finished jobs go first. Called with the lock held. +func (b *jobBook) forgetOldest() { + if len(b.jobs) <= keptJobs { + return + } + all := make([]*Job, 0, len(b.jobs)) + for _, j := range b.jobs { + all = append(all, j) + } + sort.Slice(all, func(i, k int) bool { return all[i].Started.Before(all[k].Started) }) + for _, j := range all { + if len(b.jobs) <= keptJobs { + return + } + if !j.Running { + delete(b.jobs, j.ID) + } + } +} + +// awaitJob waits up to d for a job to finish and answers a copy of it as it then stands. +func awaitJob(j *Job, d time.Duration) Job { + select { + case <-j.done: + case <-time.After(d): + } + return snapshot(j) +} + +func snapshot(j *Job) Job { + jobs.mu.Lock() + defer jobs.mu.Unlock() + c := *j + c.done = nil + return c +} + +// jobByID answers a job by its id, or says it is not known to this process. +func jobByID(id string) (Job, error) { + jobs.mu.Lock() + j, ok := jobs.jobs[id] + jobs.mu.Unlock() + if !ok { + return Job{}, fmt.Errorf("no job %s in this process: it was never started here, was forgotten after %d newer ones, or the bundle has restarted since", id, keptJobs) + } + return snapshot(j), nil +} + +// listJobs answers every job this process knows, newest first. +func listJobs() []Job { + jobs.mu.Lock() + all := make([]*Job, 0, len(jobs.jobs)) + for _, j := range jobs.jobs { + all = append(all, j) + } + jobs.mu.Unlock() + sort.Slice(all, func(i, k int) bool { return all[i].Started.After(all[k].Started) }) + out := make([]Job, 0, len(all)) + for _, j := range all { + out = append(out, snapshot(j)) + } + return out +} + +// actAsJob starts c and answers the job once it finishes or JobWait passes, whichever is first. +// A finished job that failed is answered as an error, so a failed act is never read as success. +func actAsJob(c Cmd) (Job, error) { + j := awaitJob(startJob(c), JobWait) + if !j.Running && j.Error != "" { + return j, fmt.Errorf("%s (job %s)", j.Error, j.ID) + } + return j, nil +} diff --git a/modules/docker-compose/cmd/docker-compose-tools/jobs_test.go b/modules/docker-compose/cmd/docker-compose-tools/jobs_test.go new file mode 100644 index 0000000..46e1725 --- /dev/null +++ b/modules/docker-compose/cmd/docker-compose-tools/jobs_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +func TestJobsAFastActIsAnsweredFinishedAndAFailedOneAsAnError(t *testing.T) { + using(t, func(line string, c Cmd) Result { + if c.Timeout != JobLimit { + t.Errorf("a job is held to JobLimit, not %s", c.Timeout) + } + if strings.Contains(line, "bad") { + return Result{Status: 2, Stderr: "it broke"} + } + return ok("done") + }) + j, err := actAsJob(Cmd{Name: "good"}) + if err != nil || j.Running || j.Status == nil || *j.Status != 0 || j.Output != "done" { + t.Fatalf("%+v %v", j, err) + } + if _, err := actAsJob(Cmd{Name: "bad"}); err == nil || !strings.Contains(err.Error(), "it broke") { + t.Fatalf("a failed job: %v", err) + } + got, err := jobByID(j.ID) + if err != nil || got.ID != j.ID { + t.Fatalf("by id: %+v %v", got, err) + } + if _, err := jobByID("nope"); err == nil { + t.Fatal("an unknown job") + } + if len(listJobs()) < 2 { + t.Fatal("listed") + } +} + +func TestJobsASlowActIsAnsweredRunningWithItsID(t *testing.T) { + release := make(chan struct{}) + using(t, func(line string, c Cmd) Result { <-release; return ok("") }) + j := awaitJob(startJob(Cmd{Name: "slow"}), 50*time.Millisecond) + if !j.Running || j.ID == "" { + t.Fatalf("%+v", j) + } + close(release) + time.Sleep(50 * time.Millisecond) + got, _ := jobByID(j.ID) + if got.Running { + t.Fatalf("finished afterwards: %+v", got) + } +} diff --git a/modules/docker-compose/cmd/docker-compose-tools/kit.go b/modules/docker-compose/cmd/docker-compose-tools/kit.go new file mode 100644 index 0000000..adc5aac --- /dev/null +++ b/modules/docker-compose/cmd/docker-compose-tools/kit.go @@ -0,0 +1,352 @@ +package main + +// kit.go is the same file in each of the workstations' tool bundles (fonts, docker-compose, snapd, +// flatpak, cups, bluetooth, xclip, dmenu): how a tool runs a command, escalates, bounds what it +// keeps, and names a failure. A module is built from its own directory, so the file is copied rather +// than shared; a change to one copy is made to all eight. +// +// The rules it holds (novox/hq research 026/05, to-be 38 WP4): +// - the node's tool runtime runs as the operator account, not root (ADR 0175 §4); a command that +// needs root goes through `sudo -n`, never a prompt, and a refusal is named as such; +// - one command gets 20 s, below the runtime's 30 s call limit, and is ended with everything it +// started when it takes longer; +// - each stream is kept to 256 KiB, and the answer says when it was cut; +// - a failure is an error with what went wrong in it, never an empty answer. + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "syscall" + "time" +) + +// Bounds every command is held to. +const ( + CallTimeout = 20 * time.Second + MostOutput = 256 << 10 +) + +// Cmd is one command a tool runs. +type Cmd struct { + Name string + Args []string + // Stdin is written to the command's standard input when not empty. + Stdin string + // Env is added to this process's own environment. + Env []string + // Root says the command needs root: it is run through `sudo -n` when this process is not root. + Root bool + // Timeout replaces CallTimeout; only a background job (jobs.go) asks for longer. + Timeout time.Duration + // Detached is for a program that forks a child which outlives it, as xclip does to keep the + // selection: its streams go to files, because a pipe the child inherits would hold the call open + // until the child exits. + Detached bool +} + +// Result is what a command did. +type Result struct { + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + Status int `json:"status"` + // Error is why it did not run to an answer: "not-found" when the program is not there, + // "timeout" when it was ended for taking too long, else the spawn error. + Error string `json:"error,omitempty"` + Truncated bool `json:"truncated,omitempty"` +} + +// Runner runs a command. Tests replace it; nothing else does. +type Runner func(Cmd) Result + +var ( + run Runner = execRun + euid = os.Geteuid +) + +// argv is the command as it is run: through sudo without a prompt when it needs root and this +// process is not root. +func argv(c Cmd) (string, []string) { + if c.Root && euid() != 0 { + return "sudo", append([]string{"-n", c.Name}, c.Args...) + } + return c.Name, c.Args +} + +// bounded keeps the first MostOutput bytes written to it and notes that more came. +type bounded struct { + b bytes.Buffer + cut bool +} + +func (w *bounded) Write(p []byte) (int, error) { + room := MostOutput - w.b.Len() + if room <= 0 { + w.cut = w.cut || len(p) > 0 + return len(p), nil + } + if len(p) > room { + w.b.Write(p[:room]) + w.cut = true + return len(p), nil + } + return w.b.Write(p) +} + +func execRun(c Cmd) Result { + timeout := c.Timeout + if timeout <= 0 { + timeout = CallTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + name, args := argv(c) + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = append(append(os.Environ(), "LC_ALL=C"), c.Env...) + if !c.Detached { + // Its own process group, so that ending it on a timeout ends what it started too. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if cmd.Process != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } + return nil + } + } + cmd.WaitDelay = 2 * time.Second + if c.Stdin != "" { + cmd.Stdin = strings.NewReader(c.Stdin) + } + var out, errs bounded + var outFile, errFile *os.File + if c.Detached { + var err error + if outFile, err = os.CreateTemp("", "mesh-tool-out-*"); err != nil { + return Result{Status: 127, Error: err.Error()} + } + defer os.Remove(outFile.Name()) + defer outFile.Close() + if errFile, err = os.CreateTemp("", "mesh-tool-err-*"); err != nil { + return Result{Status: 127, Error: err.Error()} + } + defer os.Remove(errFile.Name()) + defer errFile.Close() + cmd.Stdout, cmd.Stderr = outFile, errFile + } else { + cmd.Stdout, cmd.Stderr = &out, &errs + } + err := cmd.Run() + if c.Detached { + for _, f := range []struct { + file *os.File + into *bounded + }{{outFile, &out}, {errFile, &errs}} { + if _, e := f.file.Seek(0, io.SeekStart); e == nil { + _, _ = io.Copy(f.into, f.file) + } + } + } + r := Result{Stdout: out.b.String(), Stderr: errs.b.String(), Truncated: out.cut || errs.cut} + var exit *exec.ExitError + switch { + case err == nil: + case ctx.Err() == context.DeadlineExceeded: + r.Status, r.Error = 124, "timeout" + case errors.Is(err, exec.ErrNotFound) || errors.Is(err, os.ErrNotExist): + r.Status, r.Error = 127, "not-found" + case errors.As(err, &exit): + r.Status = exit.ExitCode() + default: + r.Status, r.Error = 127, err.Error() + } + return r +} + +// call runs a command and answers its result, or an error naming what went wrong. +func call(c Cmd) (Result, error) { + r := run(c) + if r.Status == 0 && r.Error == "" { + return r, nil + } + return r, failure(c, r) +} + +// failure names how a command failed: not installed, refused escalation, too slow, or its exit +// status with the end of what it said. +func failure(c Cmd, r Result) error { + program, _ := argv(c) + switch { + case r.Error == "not-found" && program == "sudo": + return fmt.Errorf("%s needs root, and sudo is not installed here for the runtime's account to escalate with", c.Name) + case r.Error == "not-found": + if hint, ok := providedBy[c.Name]; ok { + return fmt.Errorf("%s is not installed on this machine (%s)", c.Name, hint) + } + return fmt.Errorf("%s is not installed on this machine", c.Name) + case r.Error == "timeout": + limit := c.Timeout + if limit <= 0 { + limit = CallTimeout + } + return fmt.Errorf("%s gave no answer within %s and was ended", c.Name, limit) + case r.Error != "": + return fmt.Errorf("%s did not run: %s", c.Name, r.Error) + case program == "sudo" && strings.Contains(r.Stderr, "command not found"): + if hint, ok := providedBy[c.Name]; ok { + return fmt.Errorf("%s is not installed on this machine (%s)", c.Name, hint) + } + return fmt.Errorf("%s is not installed on this machine", c.Name) + case program == "sudo" && strings.HasPrefix(strings.TrimSpace(r.Stderr), "sudo:"): + return fmt.Errorf("%s needs root, and sudo -n refused the runtime's account: %s (the escalation is the sudo module's to declare)", + c.Name, firstLine(r.Stderr)) + } + said := tail(strings.TrimSpace(r.Stderr), 2000) + if said == "" { + said = tail(strings.TrimSpace(r.Stdout), 2000) + } + if said == "" { + said = "and said nothing" + } + return fmt.Errorf("%s %s exited %d: %s", c.Name, strings.Join(c.Args, " "), r.Status, said) +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +func tail(s string, n int) string { + if len(s) <= n { + return s + } + return "…" + s[len(s)-n:] +} + +// lines are a command's output lines, blank ones dropped. +func lines(s string) []string { + out := []string{} + for _, l := range strings.Split(s, "\n") { + if strings.TrimSpace(l) != "" { + out = append(out, strings.TrimRight(l, "\r")) + } + } + return out +} + +// Arguments, read the way a tool's JSON arguments arrive. + +func text(args map[string]any, key string) (string, error) { + v, ok := args[key] + if !ok || v == nil { + return "", fmt.Errorf("%s is required", key) + } + s, ok := v.(string) + if !ok { + return "", fmt.Errorf("%s must be a string", key) + } + if strings.TrimSpace(s) == "" { + return "", fmt.Errorf("%s must not be empty", key) + } + return s, nil +} + +func optText(args map[string]any, key, def string) (string, error) { + v, ok := args[key] + if !ok || v == nil { + return def, nil + } + s, ok := v.(string) + if !ok { + return "", fmt.Errorf("%s must be a string", key) + } + if strings.TrimSpace(s) == "" { + return def, nil + } + return s, nil +} + +// optWhole reads a whole number, defaulted, refused below least and held to most. +func optWhole(args map[string]any, key string, def, least, most int) (int, error) { + v, ok := args[key] + if !ok || v == nil { + return def, nil + } + f, ok := v.(float64) + if !ok { + if i, isInt := v.(int); isInt { + f = float64(i) + } else { + return 0, fmt.Errorf("%s must be a number", key) + } + } + if f != float64(int(f)) { + return 0, fmt.Errorf("%s must be a whole number", key) + } + n := int(f) + if n < least { + return 0, fmt.Errorf("%s must be at least %d", key, least) + } + if n > most { + n = most + } + return n, nil +} + +func optFlag(args map[string]any, key string, def bool) (bool, error) { + v, ok := args[key] + if !ok || v == nil { + return def, nil + } + b, ok := v.(bool) + if !ok { + return false, fmt.Errorf("%s must be true or false", key) + } + return b, nil +} + +func optList(args map[string]any, key string) ([]string, error) { + v, ok := args[key] + if !ok || v == nil { + return nil, nil + } + items, ok := v.([]any) + if !ok { + return nil, fmt.Errorf("%s must be a list of strings", key) + } + out := make([]string, 0, len(items)) + for _, it := range items { + s, ok := it.(string) + if !ok || strings.TrimSpace(s) == "" { + return nil, fmt.Errorf("%s must be a list of non-empty strings", key) + } + out = append(out, s) + } + return out, nil +} + +// oneOf refuses a value outside a closed set. +func oneOf(key, value string, allowed ...string) error { + for _, a := range allowed { + if value == a { + return nil + } + } + return fmt.Errorf("%s must be one of %s, not %q", key, strings.Join(allowed, ", "), value) +} + +// plainName refuses a name that could be read as an option or carries a path or a space: package, +// snap, application and printer names never do. +func plainName(key, value string) error { + if strings.HasPrefix(value, "-") || strings.ContainsAny(value, " \t\n/\\") { + return fmt.Errorf("%s %q is not a plain name", key, value) + } + return nil +} diff --git a/modules/docker-compose/cmd/docker-compose-tools/kit_test.go b/modules/docker-compose/cmd/docker-compose-tools/kit_test.go new file mode 100644 index 0000000..c5d3557 --- /dev/null +++ b/modules/docker-compose/cmd/docker-compose-tools/kit_test.go @@ -0,0 +1,147 @@ +package main + +// Tests of kit.go, the same in each workstation module. + +import ( + "strings" + "testing" + "time" +) + +// fake records the commands asked and answers each from a function of the command line. +type fake struct { + asked []Cmd + answer func(line string, c Cmd) Result +} + +func (f *fake) runner() Runner { + return func(c Cmd) Result { + f.asked = append(f.asked, c) + name, args := argv(c) + line := strings.TrimSpace(name + " " + strings.Join(args, " ")) + if f.answer == nil { + return Result{} + } + return f.answer(line, c) + } +} + +func (f *fake) lines() []string { + out := []string{} + for _, c := range f.asked { + name, args := argv(c) + out = append(out, strings.TrimSpace(name+" "+strings.Join(args, " "))) + } + return out +} + +// using installs a fake runner and a non-root uid for one test. +func using(t *testing.T, answer func(line string, c Cmd) Result) *fake { + t.Helper() + f := &fake{answer: answer} + wasRun, wasUID := run, euid + run, euid = f.runner(), func() int { return 1000 } + t.Cleanup(func() { run, euid = wasRun, wasUID }) + return f +} + +func ok(stdout string) Result { return Result{Stdout: stdout} } + +func TestKitAnActThatNeedsRootGoesThroughSudoWithoutAPromptUnlessAlreadyRoot(t *testing.T) { + was := euid + defer func() { euid = was }() + euid = func() int { return 1000 } + if name, args := argv(Cmd{Name: "x", Args: []string{"a"}, Root: true}); name != "sudo" || strings.Join(args, " ") != "-n x a" { + t.Fatalf("not root: %s %v", name, args) + } + if name, _ := argv(Cmd{Name: "x"}); name != "x" { + t.Fatalf("a read is run as the account: %s", name) + } + euid = func() int { return 0 } + if name, _ := argv(Cmd{Name: "x", Root: true}); name != "x" { + t.Fatalf("as root no sudo: %s", name) + } +} + +func TestKitAFailureIsNamedByHowItFailed(t *testing.T) { + was := euid + defer func() { euid = was }() + euid = func() int { return 1000 } + cases := []struct { + c Cmd + r Result + want string + }{ + {Cmd{Name: "nothere"}, Result{Status: 127, Error: "not-found"}, "not installed"}, + {Cmd{Name: "x", Root: true}, Result{Status: 127, Error: "not-found"}, "sudo is not installed"}, + {Cmd{Name: "x", Root: true}, Result{Status: 1, Stderr: "sudo: a password is required\n"}, "sudo -n refused"}, + {Cmd{Name: "x", Root: true}, Result{Status: 1, Stderr: "sudo: x: command not found\n"}, "x is not installed"}, + {Cmd{Name: "x"}, Result{Status: 124, Error: "timeout"}, "within 20s"}, + {Cmd{Name: "x", Args: []string{"y"}}, Result{Status: 3, Stderr: "boom\n"}, "x y exited 3: boom"}, + {Cmd{Name: "x"}, Result{Status: 3}, "said nothing"}, + } + for _, k := range cases { + err := failure(k.c, k.r) + if err == nil || !strings.Contains(err.Error(), k.want) { + t.Errorf("%+v: %v, want %q", k.r, err, k.want) + } + } +} + +func TestKitOutputIsBoundedAndSaysSo(t *testing.T) { + var w bounded + big := strings.Repeat("a", MostOutput+10) + n, _ := w.Write([]byte(big)) + if n != len(big) || w.b.Len() != MostOutput || !w.cut { + t.Fatalf("kept %d of %d, cut %v", w.b.Len(), len(big), w.cut) + } +} + +func TestKitTheRealRunnerRunsEndsAndReportsAMissingProgram(t *testing.T) { + r := execRun(Cmd{Name: "sh", Args: []string{"-c", "echo out; echo err >&2; exit 3"}}) + if r.Status != 3 || strings.TrimSpace(r.Stdout) != "out" || strings.TrimSpace(r.Stderr) != "err" { + t.Fatalf("%+v", r) + } + r = execRun(Cmd{Name: "sh", Args: []string{"-c", "sleep 5 & sleep 5"}, Timeout: 200 * time.Millisecond}) + if r.Error != "timeout" { + t.Fatalf("a slow command: %+v", r) + } + r = execRun(Cmd{Name: "no-such-program-anywhere"}) + if r.Error != "not-found" { + t.Fatalf("a missing program: %+v", r) + } + r = execRun(Cmd{Name: "cat", Stdin: "given"}) + if r.Stdout != "given" { + t.Fatalf("stdin: %+v", r) + } + start := time.Now() + r = execRun(Cmd{Name: "sh", Args: []string{"-c", "echo kept; (sleep 3 &) ; exit 0"}, Detached: true}) + if r.Status != 0 || strings.TrimSpace(r.Stdout) != "kept" || time.Since(start) > 2*time.Second { + t.Fatalf("a detached command returns when it exits, not when its child does: %+v after %s", r, time.Since(start)) + } +} + +func TestKitArgumentsAreReadStrictly(t *testing.T) { + args := map[string]any{"s": "x", "n": float64(5), "f": 1.5, "b": true, "l": []any{"a", "b"}} + if _, err := text(args, "missing"); err == nil { + t.Error("a missing required string") + } + if n, _ := optWhole(args, "n", 1, 1, 3); n != 3 { + t.Errorf("held to most: %d", n) + } + if _, err := optWhole(args, "n", 1, 6, 9); err == nil { + t.Error("below least") + } + if _, err := optWhole(args, "f", 1, 0, 9); err == nil { + t.Error("a fraction") + } + if l, _ := optList(args, "l"); len(l) != 2 { + t.Errorf("list: %v", l) + } + if b, _ := optFlag(args, "b", false); !b { + t.Error("flag") + } + if err := plainName("name", "--all"); err == nil { + t.Error("an option as a name") + } +} diff --git a/modules/docker-compose/cmd/docker-compose-tools/main.go b/modules/docker-compose/cmd/docker-compose-tools/main.go new file mode 100644 index 0000000..d283798 --- /dev/null +++ b/modules/docker-compose/cmd/docker-compose-tools/main.go @@ -0,0 +1,199 @@ +// The docker-compose module's tools (novox/hq research 027/02, 026/05): the compose projects on this +// machine, their containers, logs and rendered configuration, and bringing one up, down or round +// again by its directory. A Go bundle the node's runtime launches over stdio (ADR 0188, ADR 0193); it +// runs as the operator account, which reaches the container runtime through the docker group. +package main + +import ( + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +var providedBy = map[string]string{ + "docker": "the docker module installs the container runtime; this module adds compose to it", +} + +func main() { + if err := stdio.Serve("", tools()); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +// where is the argument schema every tool that acts on one project takes. +var where = map[string]any{ + "dir": map[string]any{"type": "string", "description": "the project's directory, absolute: where its compose file is"}, + "project": map[string]any{"type": "string", "description": "the project's name, for a project docker already knows (instead of dir)"}, +} + +func with(extra map[string]any) map[string]any { + out := map[string]any{} + for k, v := range where { + out[k] = v + } + for k, v := range extra { + out[k] = v + } + return out +} + +var servicesArg = map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "only these services (default all)"} + +func tools() []stdio.Tool { + return []stdio.Tool{ + { + Name: "docker_compose_projects", + Description: "The compose projects docker knows on this machine, running or stopped: name, status, working " + + "directory, compose files, services, and containers running of total. (r)", + Input: map[string]any{}, + Run: func(map[string]any) (any, error) { return Projects() }, + }, + { + Name: "docker_compose_ps", + Description: "One project's containers: service, state, health, exit code, image and published ports. (r)", + Input: with(nil), + Run: func(args map[string]any) (any, error) { + t, err := targetOf(args, false) + if err != nil { + return nil, err + } + return Ps(t) + }, + }, + { + Name: "docker_compose_logs", + Description: "One project's logs, the last lines of each service (default 200, at most 5000), optionally since a " + + "time (\"10m\", \"2026-10-04T12:00:00\"). Cut at 256 KiB. (r)", + Input: with(map[string]any{ + "services": servicesArg, + "tail": map[string]any{"type": "integer", "description": "lines per service (default 200, at most 5000)"}, + "since": map[string]any{"type": "string", "description": "only lines since this: a duration such as 10m or a timestamp"}, + }), + Run: func(args map[string]any) (any, error) { + t, err := targetOf(args, false) + if err != nil { + return nil, err + } + services, err := optList(args, "services") + if err != nil { + return nil, err + } + n, err := optWhole(args, "tail", 200, 1, 5000) + if err != nil { + return nil, err + } + since, err := optText(args, "since", "") + if err != nil { + return nil, err + } + return Logs(t, services, n, since) + }, + }, + { + Name: "docker_compose_config", + Description: "A project's configuration as compose renders it: files merged, variables filled. Values of " + + "environment variables, build arguments and labels whose names suggest a secret are replaced with " + + "[redacted]. An invalid file is answered as the error compose gives. (r)", + Input: with(nil), + Run: func(args map[string]any) (any, error) { + t, err := targetOf(args, true) + if err != nil { + return nil, err + } + return Config(t) + }, + }, + { + Name: "docker_compose_up", + Description: "Bring a project up, detached: create and start its containers, building or pulling what is " + + "missing. Answers when finished, or after 18 s with a job to follow with docker_compose_job. (a)", + Input: with(map[string]any{ + "services": servicesArg, + "build": map[string]any{"type": "boolean", "description": "build images before starting (--build)"}, + "pull": map[string]any{"type": "string", "enum": []string{"missing", "always", "never"}, "description": "pull policy (default missing)"}, + }), + Run: func(args map[string]any) (any, error) { + t, err := targetOf(args, true) + if err != nil { + return nil, err + } + services, err := optList(args, "services") + if err != nil { + return nil, err + } + build, err := optFlag(args, "build", false) + if err != nil { + return nil, err + } + pull, err := optText(args, "pull", "missing") + if err != nil { + return nil, err + } + return Up(t, services, build, pull) + }, + }, + { + Name: "docker_compose_down", + Description: "Stop and remove a project's containers and networks. Its volumes are kept: removing data is not " + + "this tool's. Answers when finished, or with a job to follow. (a)", + Input: with(nil), + Run: func(args map[string]any) (any, error) { + t, err := targetOf(args, false) + if err != nil { + return nil, err + } + return Down(t) + }, + }, + { + Name: "docker_compose_restart", + Description: "Restart a project's containers, or some of its services. Answers when finished, or with a job to follow. (a)", + Input: with(map[string]any{"services": servicesArg}), + Run: func(args map[string]any) (any, error) { + t, err := targetOf(args, false) + if err != nil { + return nil, err + } + services, err := optList(args, "services") + if err != nil { + return nil, err + } + return Restart(t, services) + }, + }, + { + Name: "docker_compose_pull", + Description: "Pull a project's images, or some services', without starting anything. Answers when finished, or with a job to follow. (a)", + Input: with(map[string]any{"services": servicesArg}), + Run: func(args map[string]any) (any, error) { + t, err := targetOf(args, true) + if err != nil { + return nil, err + } + services, err := optList(args, "services") + if err != nil { + return nil, err + } + return Pull(t, services) + }, + }, + { + Name: "docker_compose_job", + Description: "A long act this module started (up, down, restart, pull): running or finished, its exit status " + + "and the end of its output. Without job, every act this process knows, newest first. (r)", + Input: map[string]any{"job": map[string]any{"type": "string", "description": "the job id an act answered"}}, + Run: func(args map[string]any) (any, error) { + id, err := optText(args, "job", "") + if err != nil { + return nil, err + } + if id == "" { + return map[string]any{"jobs": listJobs()}, nil + } + return jobByID(id) + }, + }, + } +} diff --git a/modules/docker-compose/cmd/docker-compose-tools/manifest_kit_test.go b/modules/docker-compose/cmd/docker-compose-tools/manifest_kit_test.go new file mode 100644 index 0000000..3e675b4 --- /dev/null +++ b/modules/docker-compose/cmd/docker-compose-tools/manifest_kit_test.go @@ -0,0 +1,107 @@ +package main + +// manifest_kit_test.go is the same file in each workstation module: it reads the module's +// definition so the module's own tests can hold it to what it says. + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +type manifest struct { + Module string `json:"module"` + Capabilities []string `json:"capabilities"` + Claims []any `json:"claims"` + Seats []any `json:"seats"` + Tools []string `json:"tools"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` +} + +func readManifest(t *testing.T) manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("module.json: %v", err) + } + return m +} + +func (m manifest) resource(id string) map[string]any { + for _, r := range m.Resources { + if r["id"] == id { + return r + } + } + return nil +} + +// packages are the packages the module installs, sorted. +func (m manifest) packages() []string { + out := []string{} + for _, r := range m.Resources { + if r["type"] == "package" && r["absent"] != true { + out = append(out, r["package"].(string)) + } + } + sort.Strings(out) + return out +} + +// services are the units the module declares, by unit name. +func (m manifest) services() map[string]map[string]any { + out := map[string]map[string]any{} + for _, r := range m.Resources { + if r["type"] == "service" { + out[r["unit"].(string)] = r + } + } + return out +} + +// holdsTheBundle holds the manifest to the Go bundle this directory builds: every tool registered +// is listed and nothing else, each named _…, and the artifact builds this command. +func holdsTheBundle(t *testing.T, m manifest, prefix string) { + t.Helper() + registered := []string{} + for _, tool := range tools() { + registered = append(registered, tool.Name) + if !strings.HasPrefix(tool.Name, prefix+"_") { + t.Errorf("tool %s is not named %s_…", tool.Name, prefix) + } + if tool.Description == "" || tool.Run == nil || tool.Input == nil { + t.Errorf("tool %s is not described, runnable and given an input schema", tool.Name) + } + } + if strings.Join(registered, ",") != strings.Join(m.Tools, ",") { + t.Errorf("registered %v, listed %v", registered, m.Tools) + } + if len(m.Build.Artifacts) != 1 { + t.Fatalf("one artifact, got %d", len(m.Build.Artifacts)) + } + cwd, _ := os.Getwd() + binary := filepath.Base(cwd) + a := m.Build.Artifacts[0] + want := map[string]any{"kind": "bundle", "language": "go", "system": "arch", "from": "cmd/" + binary, "binary": binary} + for k, v := range want { + if a[k] != v { + t.Errorf("artifact %s = %v, want %v", k, a[k], v) + } + } + if loads, _ := a["loads"].([]any); len(loads) != 1 || loads[0] != binary { + t.Errorf("artifact loads %v, want [%s]", a["loads"], binary) + } + if m.Claims != nil || m.Seats != nil { + t.Errorf("claims %v, seats %v: this module holds no seat", m.Claims, m.Seats) + } +} diff --git a/modules/docker-compose/go.mod b/modules/docker-compose/go.mod new file mode 100644 index 0000000..f6fd588 --- /dev/null +++ b/modules/docker-compose/go.mod @@ -0,0 +1,5 @@ +module docker-compose + +go 1.22 + +require git.novox.be/novox/mesh-sdk/go v0.1.6 diff --git a/modules/docker-compose/go.sum b/modules/docker-compose/go.sum new file mode 100644 index 0000000..0dd6061 --- /dev/null +++ b/modules/docker-compose/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ= +git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/docker-compose/module.json b/modules/docker-compose/module.json new file mode 100644 index 0000000..2f576b4 --- /dev/null +++ b/modules/docker-compose/module.json @@ -0,0 +1,40 @@ +{ + "module": "docker-compose", + "version": "1", + "capabilities": [ + "package-manager" + ], + "tools": [ + "docker_compose_projects", + "docker_compose_ps", + "docker_compose_logs", + "docker_compose_config", + "docker_compose_up", + "docker_compose_down", + "docker_compose_restart", + "docker_compose_pull", + "docker_compose_job" + ], + "resources": [ + { + "id": "package", + "type": "package", + "package": "docker-compose" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/docker-compose-tools", + "binary": "docker-compose-tools", + "loads": [ + "docker-compose-tools" + ] + } + ] + } +}