package builder import ( "context" "os/exec" "strings" "syscall" "time" ) // A build killed by hand (novox/hq ADR 0219). // // A build is a tree of commands — git, then docker, and docker's own children — and a container the // docker client started keeps running when the client dies. So ending one by hand is three things: // the build's context is cancelled, which kills each command's whole process group rather than the // one process the context knows; every container the build started carries the build's id as a // label, so what outlived its client is found and removed by that label; and the holder announces // the outcome itself, since nothing else will. // BuildLabel is the label every container a build starts carries, valued with the build's id. const BuildLabel = "mesh.build" // KillWait is how long a command killed with its build may take to let go of its output before it // is abandoned: a grandchild holding the pipe open must not hold the build open with it. const KillWait = 10 * time.Second // inItsOwnGroup makes a command the leader of its own process group, killed as a group when its // context ends. func inItsOwnGroup(cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmd.Cancel = func() error { if cmd.Process == nil { return nil } // The negative pid is the group: the command and everything it started. if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { return cmd.Process.Kill() } return nil } cmd.WaitDelay = KillWait } // Labelled is a Runner that marks every container a build starts with the build's id, so a kill // finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command // passes rather than at each `docker run` the builder composes: a run added later is labelled too. func Labelled(run Runner, id string) Runner { return func(ctx context.Context, dir string, name string, args ...string) (string, error) { return run(ctx, dir, name, LabelledArgs(name, args, id)...) } } // LabelledArgs is a command's arguments with the build's label added, when it is a `docker run`. func LabelledArgs(name string, args []string, id string) []string { if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" { return args } out := make([]string, 0, len(args)+2) out = append(out, "run", "--label", BuildLabel+"="+id) return append(out, args[1:]...) } // RemoveContainersOf removes every container labelled with the build's id, running or not, and // says how many. Run with a context of its own: the build's is the one that was just cancelled. func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error) { out, err := run(ctx, "", "docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id) if err != nil { return 0, err } ids := strings.Fields(out) if len(ids) == 0 { return 0, nil } if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil { return 0, err } return len(ids), nil } // A holder that stops mid-build leaves its containers running (novox/hq issue 306). // // A build's containers are removed by the build itself, by a kill, or by the next delivery of the same // ask — but only on the machine that delivery reaches. A holder recreated by a rollout while a check // ran left the check's throwaway store and bus running, the ask went to another machine, and nothing // on the first one ever looked at them again: eleven pairs across four machines in two days, each a // postgres that had written its share of the disk. A holder starting has taken nothing yet, so every // container labelled with an ask of the seat is one an earlier holder here left, and is removed then. // RemoveLeftBehind removes every container on this machine labelled with an ask of the build seat — // an id of the seat's shape, `build-` — and says how many. Called by a holder before it takes // anything, so none of them can be a build it runs. A container labelled with any other id — a check // a person runs by hand (`check-here-…`), a test's — is not the seat's, and is left. func RemoveLeftBehind(ctx context.Context, run Runner) (int, error) { out, err := run(ctx, "", "docker", "ps", "-a", "--filter", "label="+BuildLabel, "--format", `{{.ID}} {{.Label "`+BuildLabel+`"}}`) if err != nil { return 0, err } var ids []string for _, line := range strings.Split(out, "\n") { fields := strings.Fields(line) if len(fields) == 2 && anAskOfTheSeat(fields[1]) { ids = append(ids, fields[0]) } } if len(ids) == 0 { return 0, nil } if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil { return 0, err } return len(ids), nil } // anAskOfTheSeat is whether an id is of the shape the controller gives the seat's asks: `build-` and the // moment it was asked, in nanoseconds. func anAskOfTheSeat(id string) bool { n, ok := strings.CutPrefix(id, "build-") if !ok || n == "" { return false } for _, c := range n { if c < '0' || c > '9' { return false } } return true }