package builder import ( "context" "os/exec" "regexp" "strings" "syscall" "time" ) // A build killed by hand (novox/hq ADR 0219). // // A build is a tree of commands — git, then docker, and docker's own children — and a container the // docker client started keeps running when the client dies. So ending one by hand is three things: // the build's context is cancelled, which kills each command's whole process group rather than the // one process the context knows; every container the build started carries the build's id as a // label, so what outlived its client is found and removed by that label; and the holder announces // the outcome itself, since nothing else will. // BuildLabel is the label every container a build starts carries, valued with the build's id. const BuildLabel = "mesh.build" // KillWait is how long a command killed with its build may take to let go of its output before it // is abandoned: a grandchild holding the pipe open must not hold the build open with it. const KillWait = 10 * time.Second // inItsOwnGroup makes a command the leader of its own process group, killed as a group when its // context ends. func inItsOwnGroup(cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmd.Cancel = func() error { if cmd.Process == nil { return nil } // The negative pid is the group: the command and everything it started. if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { return cmd.Process.Kill() } return nil } cmd.WaitDelay = KillWait } // CgroupParent is the slice every container a build or check starts joins (novox/hq issue 1798), so the node's own // weights and cap bind it: the build-agent module declares the slice with the machine's CPU weight, IO weight and // memory limits, and names it in MESH_BUILD_CGROUP_PARENT. Empty, a container joins the runtime's default. var CgroupParent string // Labelled is a Runner that marks every container a build starts with the build's id, so a kill // finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command // passes rather than at each `docker run` the builder composes: a run added later is labelled too. func Labelled(run Runner, id string) Runner { return func(ctx context.Context, dir string, name string, args ...string) (string, error) { return run(ctx, dir, name, LabelledArgs(name, args, id)...) } } // LabelledArgs is a command's arguments with the build's label added, when it is a `docker run`. func LabelledArgs(name string, args []string, id string) []string { if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" { return args } out := make([]string, 0, len(args)+4) out = append(out, "run", "--label", BuildLabel+"="+id) if CgroupParent != "" { out = append(out, "--cgroup-parent", CgroupParent) } return append(out, args[1:]...) } // RemoveContainersOf removes every container labelled with the build's id, running or not, and // says how many. Run with a context of its own: the build's is the one that was just cancelled. func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error) { out, err := run(ctx, "", "docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id) if err != nil { return 0, err } ids := strings.Fields(out) if len(ids) == 0 { return 0, nil } if err := removeWithVolumes(ctx, run, ids); err != nil { return 0, err } return len(ids), nil } // MountedVolumes is the template `docker inspect --format` reads the names of a container's volumes with. const MountedVolumes = `{{range .Mounts}}{{if eq .Type "volume"}}{{println .Name}}{{end}}{{end}}` // anonymousVolume is the name docker gives a volume it makes for an image's VOLUME line: 64 hex digits. var anonymousVolume = regexp.MustCompile(`^[0-9a-f]{64}$`) // removeWithVolumes removes containers together with their anonymous volumes (novox/hq issue 1772). A throwaway // store declares its data directory a VOLUME, docker makes an anonymous volume for it, and `docker rm -f` // without -v leaves that volume for ever: 1,629 of them, 107 GB, on one build node. So the volumes the // containers mount are read first, the containers are removed with -v, and an anonymous volume still there // afterwards is removed by name. Only a volume one of these containers mounted, and only one with an anonymous // volume's name, is touched: a named volume — an operator's own data — never is. A volume already gone is // no failure. func removeWithVolumes(ctx context.Context, run Runner, ids []string) error { var volumes []string if out, err := run(ctx, "", "docker", append([]string{"inspect", "--format", MountedVolumes}, ids...)...); err == nil { seen := map[string]bool{} for _, name := range strings.Fields(out) { if anonymousVolume.MatchString(name) && !seen[name] { seen[name] = true volumes = append(volumes, name) } } } if _, err := run(ctx, "", "docker", append([]string{"rm", "-f", "-v"}, ids...)...); err != nil { return err } if len(volumes) > 0 { _, _ = run(ctx, "", "docker", append([]string{"volume", "rm"}, volumes...)...) } return nil } // A holder that stops mid-build leaves its containers running (novox/hq issue 306). // // A build's containers are removed by the build itself, by a kill, or by the next delivery of the same // ask — but only on the machine that delivery reaches. A holder recreated by a rollout while a check // ran left the check's throwaway store and bus running, the ask went to another machine, and nothing // on the first one ever looked at them again: eleven pairs across four machines in two days, each a // postgres that had written its share of the disk. A holder starting has taken nothing yet, so every // container labelled with an ask of the seat is one an earlier holder here left, and is removed then. // RemoveLeftBehind removes every container on this machine labelled with an ask of the build seat — // an id of the seat's shape, `build-` — and says how many. Called by a holder before it takes // anything, so none of them can be a build it runs. A container labelled with any other id — a check // a person runs by hand (`check-here-…`), a test's — is not the seat's, and is left. func RemoveLeftBehind(ctx context.Context, run Runner) (int, error) { out, err := run(ctx, "", "docker", "ps", "-a", "--filter", "label="+BuildLabel, "--format", `{{.ID}} {{.Label "`+BuildLabel+`"}}`) if err != nil { return 0, err } var ids []string for _, line := range strings.Split(out, "\n") { fields := strings.Fields(line) if len(fields) == 2 && anAskOfTheSeat(fields[1]) { ids = append(ids, fields[0]) } } if len(ids) == 0 { return 0, nil } if err := removeWithVolumes(ctx, run, ids); err != nil { return 0, err } return len(ids), nil } // anAskOfTheSeat is whether an id is of the shape the controller gives the seat's asks: `build-` and the // moment it was asked, in nanoseconds. func anAskOfTheSeat(id string) bool { n, ok := strings.CutPrefix(id, "build-") if !ok || n == "" { return false } for _, c := range n { if c < '0' || c > '9' { return false } } return true }