mesh-controller/internal/builder/kill.go

package builder

import (
	"context"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
	"time"
)

// A build killed by hand (novox/hq ADR 0219).
//
// A build is a tree of commands — git, then docker, and docker's own children — and a container the
// docker client started keeps running when the client dies. So ending one by hand is three things:
// the build's context is cancelled, which kills each command's whole process group rather than the
// one process the context knows; every container the build started carries the build's id as a
// label, so what outlived its client is found and removed by that label; and the holder announces
// the outcome itself, since nothing else will.

// BuildLabel is the label every container a build starts carries, valued with the build's id.
const BuildLabel = "mesh.build"

// KillWait is how long a command killed with its build may take to let go of its output before it
// is abandoned: a grandchild holding the pipe open must not hold the build open with it.
const KillWait = 10 * time.Second

// inItsOwnGroup makes a command the leader of its own process group, killed as a group when its
// context ends.
func inItsOwnGroup(cmd *exec.Cmd) {
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	cmd.Cancel = func() error {
		if cmd.Process == nil {
			return nil
		}
		// The negative pid is the group: the command and everything it started.
		if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil {
			return cmd.Process.Kill()
		}
		return nil
	}
	cmd.WaitDelay = KillWait
}

// Labelled is a Runner that marks every container a build starts with the build's id, so a kill
// finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command
// passes rather than at each `docker run` the builder composes: a run added later is labelled too.
func Labelled(run Runner, id string) Runner {
	return func(ctx context.Context, dir string, name string, args ...string) (string, error) {
		return run(ctx, dir, name, LabelledArgs(name, args, id)...)
	}
}

// LabelledArgs is a command's arguments with the build's label added, when it is a `docker run`.
func LabelledArgs(name string, args []string, id string) []string {
	if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" {
		return args
	}
	out := make([]string, 0, len(args)+2)
	out = append(out, "run", "--label", BuildLabel+"="+id)
	return append(out, args[1:]...)
}

// RemoveContainersOf removes every container labelled with the build's id, running or not, and
// says how many. Run with a context of its own: the build's is the one that was just cancelled.
func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error) {
	out, err := run(ctx, "", "docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id)
	if err != nil {
		return 0, err
	}
	ids := strings.Fields(out)
	if len(ids) == 0 {
		return 0, nil
	}
	if err := removeWithVolumes(ctx, run, ids); err != nil {
		return 0, err
	}
	return len(ids), nil
}

// MountedVolumes is the template `docker inspect --format` reads the names of a container's volumes with.
const MountedVolumes = `{{range .Mounts}}{{if eq .Type "volume"}}{{println .Name}}{{end}}{{end}}`

// anonymousVolume is the name docker gives a volume it makes for an image's VOLUME line: 64 hex digits.
var anonymousVolume = regexp.MustCompile(`^[0-9a-f]{64}$`)

// removeWithVolumes removes containers together with their anonymous volumes (novox/hq issue 1772). A throwaway
// store declares its data directory a VOLUME, docker makes an anonymous volume for it, and `docker rm -f`
// without -v leaves that volume for ever: 1,629 of them, 107 GB, on one build node. So the volumes the
// containers mount are read first, the containers are removed with -v, and an anonymous volume still there
// afterwards is removed by name. Only a volume one of these containers mounted, and only one with an anonymous
// volume's name, is touched: a named volume — an operator's own data — never is. A volume already gone is
// no failure.
func removeWithVolumes(ctx context.Context, run Runner, ids []string) error {
	var volumes []string
	if out, err := run(ctx, "", "docker", append([]string{"inspect", "--format", MountedVolumes}, ids...)...); err == nil {
		seen := map[string]bool{}
		for _, name := range strings.Fields(out) {
			if anonymousVolume.MatchString(name) && !seen[name] {
				seen[name] = true
				volumes = append(volumes, name)
			}
		}
	}
	if _, err := run(ctx, "", "docker", append([]string{"rm", "-f", "-v"}, ids...)...); err != nil {
		return err
	}
	if len(volumes) > 0 {
		_, _ = run(ctx, "", "docker", append([]string{"volume", "rm"}, volumes...)...)
	}
	return nil
}

// A holder that stops mid-build leaves its containers running (novox/hq issue 306).
//
// A build's containers are removed by the build itself, by a kill, or by the next delivery of the same
// ask — but only on the machine that delivery reaches. A holder recreated by a rollout while a check
// ran left the check's throwaway store and bus running, the ask went to another machine, and nothing
// on the first one ever looked at them again: eleven pairs across four machines in two days, each a
// postgres that had written its share of the disk. A holder starting has taken nothing yet, so every
// container labelled with an ask of the seat is one an earlier holder here left, and is removed then.

// RemoveLeftBehind removes every container on this machine labelled with an ask of the build seat —
// an id of the seat's shape, `build-<n>` — and says how many. Called by a holder before it takes
// anything, so none of them can be a build it runs. A container labelled with any other id — a check
// a person runs by hand (`check-here-…`), a test's — is not the seat's, and is left.
func RemoveLeftBehind(ctx context.Context, run Runner) (int, error) {
	out, err := run(ctx, "", "docker", "ps", "-a", "--filter", "label="+BuildLabel,
		"--format", `{{.ID}} {{.Label "`+BuildLabel+`"}}`)
	if err != nil {
		return 0, err
	}
	var ids []string
	for _, line := range strings.Split(out, "\n") {
		fields := strings.Fields(line)
		if len(fields) == 2 && anAskOfTheSeat(fields[1]) {
			ids = append(ids, fields[0])
		}
	}
	if len(ids) == 0 {
		return 0, nil
	}
	if err := removeWithVolumes(ctx, run, ids); err != nil {
		return 0, err
	}
	return len(ids), nil
}

// anAskOfTheSeat is whether an id is of the shape the controller gives the seat's asks: `build-` and the
// moment it was asked, in nanoseconds.
func anAskOfTheSeat(id string) bool {
	n, ok := strings.CutPrefix(id, "build-")
	if !ok || n == "" {
		return false
	}
	for _, c := range n {
		if c < '0' || c > '9' {
			return false
		}
	}
	return true
}