Let a build agent be paused, have a build killed, and end an ask cancelled as it took it (hq ADR 0219)
A queued ask could only be waited out and a running build only ended by stopping the machine, which redelivered it elsewhere. The holder now serves current, kill, pause and resume on its own machine's subjects; a kill ends the build's process group and labelled containers and settles the ask as failed, killed by hand; pause is kept in the workspace across a restart and said on the bus. The controller writes cancelled ids to a cancelled set the holder reads on taking an ask, closing the race a delete alone leaves.
This commit is contained in:
jochen
@@ -0,0 +1,295 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
|
||||
//
|
||||
// **The queue is the controller's; the build running here is this machine's.** The controller can
|
||||
// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
|
||||
// on this machine and containers in this machine's runtime, and only this machine can end them. So
|
||||
// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
|
||||
// it, pause, resume.
|
||||
//
|
||||
// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
|
||||
// set per build — so "the build running here" is one or none, and kill names it by id so a call
|
||||
// that arrives as one build ends and the next begins cannot end the wrong one.
|
||||
|
||||
// holder is this machine's state as a holder of the build seat.
|
||||
type holder struct {
|
||||
on, seat string
|
||||
// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
|
||||
// rather than silently taking work again.
|
||||
workspace string
|
||||
// say publishes this machine's state: whether it takes work (link.HolderState).
|
||||
say func(link.HolderState) error
|
||||
// remove runs what a kill needs outside the build: the containers left behind.
|
||||
remove builder.Runner
|
||||
|
||||
mu sync.Mutex
|
||||
paused bool
|
||||
running *running
|
||||
}
|
||||
|
||||
// running is the build this machine is doing.
|
||||
type running struct {
|
||||
request link.BuildRequest
|
||||
step string
|
||||
started time.Time
|
||||
cancel context.CancelFunc
|
||||
killed bool
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// pausedFile is where the flag lives in the workspace.
|
||||
func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
|
||||
|
||||
// newHolder reads the paused flag the workspace keeps.
|
||||
func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
|
||||
h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
|
||||
if _, err := os.Stat(pausedFile(workspace)); err == nil {
|
||||
h.paused = true
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Paused is asked by the taking loop before every fetch.
|
||||
func (h *holder) Paused() bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.paused
|
||||
}
|
||||
|
||||
// setPaused records the flag in the workspace first and then in memory, so what this holder says
|
||||
// it is and what it would be after a restart never differ.
|
||||
func (h *holder) setPaused(paused bool) error {
|
||||
path := pausedFile(h.workspace)
|
||||
if paused {
|
||||
if err := os.MkdirAll(h.workspace, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
|
||||
}
|
||||
} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.paused = paused
|
||||
h.mu.Unlock()
|
||||
h.announce()
|
||||
return nil
|
||||
}
|
||||
|
||||
// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
|
||||
// controller reading an older state is a plan read as late rather than a build lost.
|
||||
func (h *holder) announce() {
|
||||
if h.say == nil {
|
||||
return
|
||||
}
|
||||
if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
|
||||
// a pause outlives the events stream's retention.
|
||||
func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||
h.announce()
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if h.Paused() {
|
||||
h.announce()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// begin records the build this machine took, with the cancel that ends it.
|
||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||
h.mu.Lock()
|
||||
h.running = r
|
||||
h.mu.Unlock()
|
||||
return r
|
||||
}
|
||||
|
||||
// end clears it, and lets a kill waiting on it know it is over.
|
||||
func (h *holder) end(r *running) {
|
||||
h.mu.Lock()
|
||||
if h.running == r {
|
||||
h.running = nil
|
||||
}
|
||||
h.mu.Unlock()
|
||||
close(r.done)
|
||||
}
|
||||
|
||||
// stepped records the step a build is at, for `current`.
|
||||
func (h *holder) stepped(r *running, step string) {
|
||||
h.mu.Lock()
|
||||
r.step = step
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// wasKilled is whether a person killed this build.
|
||||
func (h *holder) wasKilled(r *running) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return r.killed
|
||||
}
|
||||
|
||||
// currentBuild is what `current` answers.
|
||||
type currentBuild struct {
|
||||
On string `json:"on"`
|
||||
Paused bool `json:"paused"`
|
||||
Running *struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
} `json:"running,omitempty"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
func (h *holder) current() currentBuild {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
out := currentBuild{On: h.on, Paused: h.paused}
|
||||
taking := "taking builds"
|
||||
if h.paused {
|
||||
taking = "paused, taking no new build"
|
||||
}
|
||||
if h.running == nil {
|
||||
out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
|
||||
return out
|
||||
}
|
||||
r := h.running
|
||||
elapsed := time.Since(r.started).Round(time.Second)
|
||||
out.Running = &struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
|
||||
r.started.UTC().Format(time.RFC3339), elapsed.String()}
|
||||
out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
|
||||
h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
|
||||
return out
|
||||
}
|
||||
|
||||
// kill ends the build with this id, if it is the one running here: its context cancelled — which
|
||||
// kills each command's process group — and the containers it started removed by their label. The
|
||||
// build's own goroutine then announces it failed, killed by hand, and settles the ask, so it is not
|
||||
// redelivered; this waits a while for that, to say it happened.
|
||||
func (h *holder) kill(id string) (string, error) {
|
||||
h.mu.Lock()
|
||||
r := h.running
|
||||
if r == nil || r.request.ID != id {
|
||||
doing := "nothing"
|
||||
if r != nil {
|
||||
doing = r.request.ID
|
||||
}
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
|
||||
}
|
||||
r.killed = true
|
||||
cancel, done := r.cancel, r.done
|
||||
h.mu.Unlock()
|
||||
|
||||
cancel()
|
||||
cleanup, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
removed, err := builder.RemoveContainersOf(cleanup, h.remove, id)
|
||||
containers := fmt.Sprintf("%d container(s) it started removed", removed)
|
||||
if err != nil {
|
||||
containers = "its containers could not be listed or removed: " + err.Error()
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as "+
|
||||
"failed, %s — settled, so it is not handed to another machine", id, r.request.Repository, h.on,
|
||||
containers, link.KilledByHand), nil
|
||||
case <-time.After(30 * time.Second):
|
||||
return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
|
||||
"its outcome is in", id, h.on, containers), nil
|
||||
}
|
||||
}
|
||||
|
||||
// handlers are the seat's verbs, as this machine answers them.
|
||||
func (h *holder) handlers() map[string]link.ToolHandler {
|
||||
return map[string]link.ToolHandler{
|
||||
"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
|
||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||
var args struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
|
||||
return nil, errors.New("kill needs the build's id")
|
||||
}
|
||||
said, err := h.kill(strings.TrimSpace(args.ID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": said}, nil
|
||||
},
|
||||
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
said := h.on + " is paused: it takes no new build until resumed"
|
||||
if c := h.current(); c.Running != nil {
|
||||
said += "; " + c.Running.ID + " runs on and finishes"
|
||||
}
|
||||
return map[string]any{"said": said, "paused": true}, nil
|
||||
},
|
||||
"resume": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func orNothing(s string) string {
|
||||
if s == "" {
|
||||
return "its start"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
|
||||
// log it would be is the one being ended.
|
||||
func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
|
||||
// 0219), and what it says about itself follows.
|
||||
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
var said []link.HolderState
|
||||
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
|
||||
said = append(said, s)
|
||||
return nil
|
||||
})
|
||||
if h.Paused() {
|
||||
t.Fatal("a new holder starts paused")
|
||||
}
|
||||
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
|
||||
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
|
||||
}
|
||||
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
|
||||
if !again.Paused() {
|
||||
t.Fatal("restarted, the holder forgot it was paused")
|
||||
}
|
||||
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
|
||||
t.Fatal("resumed, the holder came back paused")
|
||||
}
|
||||
}
|
||||
|
||||
// kill ends the build with that id — its context, which ends its commands — removes what it left by
|
||||
// label, and refuses an id it is not building.
|
||||
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||
var removed []string
|
||||
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
removed = append(removed, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "ps" {
|
||||
return "c1\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
kill := h.handlers()["kill"]
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
|
||||
t.Fatal("killed a build while none ran")
|
||||
}
|
||||
|
||||
building, cancel := context.WithCancel(context.Background())
|
||||
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
|
||||
h.stepped(r, "image")
|
||||
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
|
||||
t.Fatalf("current says %+v", c)
|
||||
}
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "build-1") {
|
||||
t.Fatalf("killed another id: %v", err)
|
||||
}
|
||||
// The build's own goroutine: it ends when its context does, as a build's commands do.
|
||||
go func() {
|
||||
<-building.Done()
|
||||
h.end(r)
|
||||
}()
|
||||
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if building.Err() == nil {
|
||||
t.Fatal("the build's context was not cancelled")
|
||||
}
|
||||
if !h.wasKilled(r) {
|
||||
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
|
||||
}
|
||||
said := answer.(map[string]any)["said"].(string)
|
||||
if !strings.Contains(said, "1 container(s)") || !strings.Contains(said, link.KilledByHand) {
|
||||
t.Errorf("kill said %q", said)
|
||||
}
|
||||
if len(removed) != 2 || !strings.Contains(removed[0], "label=mesh.build=build-1") {
|
||||
t.Errorf("removed %v", removed)
|
||||
}
|
||||
if c := h.current(); c.Running != nil {
|
||||
t.Errorf("after the kill current says %+v", c)
|
||||
}
|
||||
}
|
||||
+76
-11
@@ -19,11 +19,13 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
@@ -107,48 +109,89 @@ func run() error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
machine, err := takeWorkFrom(credential, on)
|
||||
js, seat, err := dialFor(credential)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
|
||||
// paused flag is read from the workspace before anything is taken, so a holder restarted while
|
||||
// paused takes nothing.
|
||||
h := newHolder(on, seat, workspace, func(state link.HolderState) error {
|
||||
body, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
|
||||
return err
|
||||
})
|
||||
if h.Paused() {
|
||||
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
||||
}
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
// one serves none, and a subscription its holder has no grant for would be refused for ever.
|
||||
if seat == link.TheBuildMachine {
|
||||
stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
|
||||
log.New(os.Stderr, "", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopServing()
|
||||
go h.stateWhilePaused(ctx, time.Hour)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
answer(ctx, publisher, on, workspace, work)
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
|
||||
// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
|
||||
// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
|
||||
// holds.
|
||||
//
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
||||
// machine told about both would take work from one and answer on the other, and every log line would
|
||||
// say it was fine.
|
||||
func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
|
||||
func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
||||
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
||||
// and that is enough to dial it, pinned.
|
||||
if !credential.onTheNewBus() {
|
||||
return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
}
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
||||
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
||||
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
||||
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
||||
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
||||
return link.MachineOverNATSOn(js, on, seat), nil
|
||||
return js, seat, nil
|
||||
}
|
||||
|
||||
// answer does one build and says what happened, whichever way it went.
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
|
||||
request := work.Request()
|
||||
|
||||
// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
|
||||
// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
|
||||
// reaches it through the parent, and that keeps today's meaning below.
|
||||
building, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
var mine *running
|
||||
if h != nil {
|
||||
mine = h.begin(request, cancel)
|
||||
defer h.end(mine)
|
||||
}
|
||||
|
||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
||||
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
||||
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
||||
@@ -162,6 +205,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
say := func(step, message string) {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
work.Say(step, message)
|
||||
if mine != nil && step != "run" && step != "output" {
|
||||
h.stepped(mine, step)
|
||||
}
|
||||
}
|
||||
builder.Said = say
|
||||
defer func() { builder.Said = nil }()
|
||||
@@ -188,11 +234,20 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||
// after a clone that then fails at npm ci.
|
||||
built, err = builder.Build(ctx, builder.Command, publisher,
|
||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||
// docker client (ADR 0219).
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
if err != nil {
|
||||
// Only a build the kill ended: one that finished in the moment the kill arrived built, and says so.
|
||||
killed := err != nil && mine != nil && h.wasKilled(mine)
|
||||
if killed {
|
||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||
// error it ended with is the kill's consequence, not a fault of the source.
|
||||
result.Failed = link.KilledByHand
|
||||
say("failed", link.KilledByHand)
|
||||
} else if err != nil {
|
||||
// A failure is a result. A build that fails and says nothing is indistinguishable from a
|
||||
// builder that is not running, and those want completely different responses.
|
||||
result.Failed = err.Error()
|
||||
@@ -217,7 +272,17 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
}
|
||||
|
||||
if err := work.Announce(ctx, result); err != nil {
|
||||
// A killed build is announced on a context of its own: the build's was the one cancelled, and the
|
||||
// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
|
||||
// to be built again. A machine being stopped is the other case and keeps its meaning: the
|
||||
// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
|
||||
announcing := ctx
|
||||
if killed {
|
||||
fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
announcing = fresh
|
||||
}
|
||||
if err := work.Announce(announcing, result); err != nil {
|
||||
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
||||
// failed is a lie about work that was done — and the request stays unsettled below only if
|
||||
// nothing was said at all, so another machine can try.
|
||||
|
||||
Reference in New Issue
Block a user