diff --git a/cmd/mesh-builder/holder.go b/cmd/mesh-builder/holder.go new file mode 100644 index 0000000..1c3edf3 --- /dev/null +++ b/cmd/mesh-builder/holder.go @@ -0,0 +1,386 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/micro" + + "github.com/novox/mesh-controller/internal/builder" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219). +// +// **The queue is the controller's; the build running here is this machine's.** The controller can +// see and change what waits in the seat's queue, but an ask a holder already took is a process tree +// on this machine and containers in this machine's runtime, and only this machine can end them. So +// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill +// it, pause, resume. +// +// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global +// set per build — so "the build running here" is one or none, and kill names it by id so a call +// that arrives as one build ends and the next begins cannot end the wrong one. + +// holder is this machine's state as a holder of the build seat. +type holder struct { + on, seat string + // workspace is where the paused flag is kept, so a holder restarted while paused stays paused + // rather than silently taking work again. + workspace string + // say publishes this machine's state: whether it takes work (link.HolderState). + say func(link.HolderState) error + // remove runs what a kill needs outside the build: the containers left behind. + remove builder.Runner + + mu sync.Mutex + paused bool + running *running +} + +// running is the build this machine is doing. +type running struct { + request link.BuildRequest + step string + started time.Time + cancel context.CancelFunc + killed bool + // returned is the build's own work having ended, before a kill or not; outcome is what was then + // announced, and announced whether it went out. + returned bool + outcome string + announced bool + done chan struct{} +} + +// pausedFile is where the flag lives in the workspace. +func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") } + +// newHolder reads the paused flag the workspace keeps. +func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder { + h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun} + if _, err := os.Stat(pausedFile(workspace)); err == nil { + h.paused = true + } + return h +} + +// Paused is asked by the taking loop before every fetch. +func (h *holder) Paused() bool { + h.mu.Lock() + defer h.mu.Unlock() + return h.paused +} + +// setPaused records the flag in the workspace first and then in memory, so what this holder says +// it is and what it would be after a restart never differ. +func (h *holder) setPaused(paused bool) error { + path := pausedFile(h.workspace) + if paused { + if err := os.MkdirAll(h.workspace, 0o755); err != nil { + return err + } + if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil { + return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err) + } + } else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot remove the paused flag %s: %w", path, err) + } + h.mu.Lock() + h.paused = paused + h.mu.Unlock() + h.announce() + return nil +} + +// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the +// controller reading an older state is a plan read as late rather than a build lost. +func (h *holder) announce() { + if h.say == nil { + return + } + if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil { + fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err) + } +} + +// stateWhilePaused says this machine's state at start, and again every while it stays paused, so +// a pause outlives the events stream's retention. +func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) { + h.announce() + tick := time.NewTicker(every) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if h.Paused() { + h.announce() + } + } + } +} + +// begin records the build this machine took, with the cancel that ends it. +func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running { + r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})} + h.mu.Lock() + h.running = r + h.mu.Unlock() + return r +} + +// end clears it, and lets a kill waiting on it know it is over. +func (h *holder) end(r *running) { + h.mu.Lock() + if h.running == r { + h.running = nil + } + h.mu.Unlock() + close(r.done) +} + +// stepped records the step a build is at, for `current`. +func (h *holder) stepped(r *running, step string) { + h.mu.Lock() + r.step = step + h.mu.Unlock() +} + +// currentBuild is what `current` answers. +type currentBuild struct { + On string `json:"on"` + Paused bool `json:"paused"` + Running *struct { + ID string `json:"id"` + Repository string `json:"repository"` + Path string `json:"path,omitempty"` + Ref string `json:"ref,omitempty"` + Step string `json:"step,omitempty"` + Started string `json:"started"` + Elapsed string `json:"elapsed"` + } `json:"running,omitempty"` + Said string `json:"said"` +} + +func (h *holder) current() currentBuild { + h.mu.Lock() + defer h.mu.Unlock() + out := currentBuild{On: h.on, Paused: h.paused} + taking := "taking builds" + if h.paused { + taking = "paused, taking no new build" + } + if h.running == nil { + out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking) + return out + } + r := h.running + elapsed := time.Since(r.started).Round(time.Second) + out.Running = &struct { + ID string `json:"id"` + Repository string `json:"repository"` + Path string `json:"path,omitempty"` + Ref string `json:"ref,omitempty"` + Step string `json:"step,omitempty"` + Started string `json:"started"` + Elapsed string `json:"elapsed"` + }{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step, + r.started.UTC().Format(time.RFC3339), elapsed.String()} + out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s", + h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking) + return out +} + +// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between +// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer +// (killAnswer in the controller's queue.go, 75s). +var ( + killRemoves = 15 * time.Second + killWaits = 20 * time.Second +) + +// kill ends the build with this id, if it is the one running here and still working: its context +// cancelled — which kills each command's process group — and the containers it started removed by +// their label, once at once and again after the build has ended, so one created while it was being +// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the +// ask so it is not redelivered; the answer says whether that happened. +func (h *holder) kill(id string) (string, error) { + h.mu.Lock() + r := h.running + if r == nil || r.request.ID != id { + doing := "nothing" + if r != nil { + doing = r.request.ID + } + h.mu.Unlock() + return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing) + } + if r.returned { + h.mu.Unlock() + return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on) + } + r.killed = true + cancel, done := r.cancel, r.done + h.mu.Unlock() + + cancel() + removed, removeErr := h.removeContainers(id) + ended := false + select { + case <-done: + ended = true + case <-time.After(killWaits): + } + again, againErr := h.removeContainers(id) + removed += again + if removeErr == nil { + removeErr = againErr + } + containers := fmt.Sprintf("%d container(s) it started removed", removed) + if removeErr != nil { + containers = "its containers could not all be listed or removed: " + removeErr.Error() + } + if !ended { + return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+ + "its outcome is in", id, h.on, containers), nil + } + h.mu.Lock() + outcome, announced := r.outcome, r.announced + h.mu.Unlock() + if !announced { + return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+ + "announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on, + containers), nil + } + return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+ + "settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil +} + +// removeContainers is one pass of removing what the build left, bounded. +func (h *holder) removeContainers(id string) (int, error) { + cleanup, stop := context.WithTimeout(context.Background(), killRemoves) + defer stop() + return builder.RemoveContainersOf(cleanup, h.remove, id) +} + +// returned records that the build's work ended, and says whether a kill came first — only then is +// the build killed; an error it ended with on its own is its own outcome. +func (h *holder) returned(r *running) bool { + h.mu.Lock() + defer h.mu.Unlock() + r.returned = true + return r.killed +} + +// said records the outcome announced, and whether it went out, for a kill to answer with. +func (h *holder) said(r *running, outcome string, announced bool) { + h.mu.Lock() + r.outcome, r.announced = outcome, announced + h.mu.Unlock() +} + +// handlers are the seat's verbs, as this machine answers them. +func (h *holder) handlers() map[string]link.ToolHandler { + return map[string]link.ToolHandler{ + "current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil }, + "kill": func(_ context.Context, raw json.RawMessage) (any, error) { + var args struct { + ID string `json:"id"` + } + if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" { + return nil, errors.New("kill needs the build's id") + } + said, err := h.kill(strings.TrimSpace(args.ID)) + if err != nil { + return nil, err + } + return map[string]any{"said": said}, nil + }, + "pause": func(context.Context, json.RawMessage) (any, error) { + if err := h.setPaused(true); err != nil { + return nil, err + } + said := h.on + " is paused: it takes no new build until resumed" + if c := h.current(); c.Running != nil { + said += "; " + c.Running.ID + " runs on and finishes" + } + return map[string]any{"said": said, "paused": true}, nil + }, + "resume": func(context.Context, json.RawMessage) (any, error) { + if err := h.setPaused(false); err != nil { + return nil, err + } + return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil + }, + } +} + +func orNothing(s string) string { + if s == "" { + return "its start" + } + return s +} + +// plainRun runs a command outside any build: no line reaches a build's log, because the build whose +// log it would be is the one being ended. +func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) + } + return string(out), nil +} + +// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this +// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat, +// the module answering, the seat, scope node, the machine, its description and argument schema — so +// the console finds `/node-build-agent.kill` by searching, as it finds any seat's verb. +// +// One service per machine, named for the seat and identified by the machine, so the answer is this +// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no +// store, and what it serves is what this binary was built to serve. +func announcement(seat, module, node string) micro.Info { + s, _ := catalogue.SeatNamed(seat) + var endpoints []micro.EndpointInfo + for _, v := range s.Serves { + schema, _ := json.Marshal(v.Input) + endpoints = append(endpoints, micro.EndpointInfo{ + Name: seat + "__" + v.Name, + Subject: link.NodeSeatToolSubject(seat, v.Name, node), + // The queue group the verbs are served in, as every runtime announces its own. + QueueGroup: "seat." + seat, + Metadata: map[string]string{ + "kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node", + "node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema), + }, + }) + } + return micro.Info{ + ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0", + Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}}, + Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)", + Endpoints: endpoints, + } +} + +// moduleOf is the module a credential was issued for: its user is `.`. +func moduleOf(user string) string { + if _, module, ok := strings.Cut(user, "."); ok && module != "" { + return module + } + return "build-agent" +} diff --git a/cmd/mesh-builder/holder_test.go b/cmd/mesh-builder/holder_test.go new file mode 100644 index 0000000..c27e1d7 --- /dev/null +++ b/cmd/mesh-builder/holder_test.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR +// 0219), and what it says about itself follows. +func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) { + workspace := t.TempDir() + var said []link.HolderState + h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error { + said = append(said, s) + return nil + }) + if h.Paused() { + t.Fatal("a new holder starts paused") + } + if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil { + t.Fatal(err) + } + if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" { + t.Fatalf("paused: %v, said %+v", h.Paused(), said) + } + again := newHolder("ace", link.TheBuildMachine, workspace, nil) + if !again.Paused() { + t.Fatal("restarted, the holder forgot it was paused") + } + if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil { + t.Fatal(err) + } + if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() { + t.Fatal("resumed, the holder came back paused") + } +} + +// kill ends the build with that id — its context, which ends its commands — removes what it left by +// label, and refuses an id it is not building. +func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) { + h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil) + var removed []string + h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) { + removed = append(removed, name+" "+strings.Join(args, " ")) + if args[0] == "ps" { + return "c1\n", nil + } + return "", nil + } + kill := h.handlers()["kill"] + if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil { + t.Fatal("killed a build while none ran") + } + + building, cancel := context.WithCancel(context.Background()) + r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel) + h.stepped(r, "image") + if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" { + t.Fatalf("current says %+v", c) + } + if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil || + !strings.Contains(err.Error(), "build-1") { + t.Fatalf("killed another id: %v", err) + } + // The build's own goroutine: it ends when its context does, as a build's commands do, and + // announces what came of it. + go func() { + <-building.Done() + if h.returned(r) { + h.said(r, link.KilledByHand, true) + } + h.end(r) + }() + answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)) + if err != nil { + t.Fatal(err) + } + if building.Err() == nil { + t.Fatal("the build's context was not cancelled") + } + if !r.killed { + t.Error("the build is not marked killed, so it would be announced as an ordinary failure") + } + said := answer.(map[string]any)["said"].(string) + if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) { + t.Errorf("kill said %q", said) + } + // Removed at the kill and again once the build had ended: a container made in between is caught. + if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") { + t.Errorf("removed %v", removed) + } + if c := h.current(); c.Running != nil { + t.Errorf("after the kill current says %+v", c) + } +} + +// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more. +func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) { + info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace") + if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 { + t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints)) + } + kill := info.Endpoints[1] + md := kill.Metadata + if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" || + md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" || + !strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) { + t.Fatalf("kill is announced as %+v", kill) + } + body, err := json.Marshal(info) + if err != nil || len(body) > 8*1024 { + t.Fatalf("the answer is %d bytes (%v)", len(body), err) + } +} + +// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its +// own error is its outcome. One whose outcome could not be announced is not said to be settled. +func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) { + h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil) + h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil } + _, cancel := context.WithCancel(context.Background()) + r := h.begin(link.BuildRequest{ID: "build-1"}, cancel) + if killed := h.returned(r); killed { + t.Fatal("a build nobody killed reads as killed") + } + if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") { + t.Fatalf("killed a build that had ended: %v", err) + } + h.end(r) + + building, cancel := context.WithCancel(context.Background()) + r = h.begin(link.BuildRequest{ID: "build-2"}, cancel) + go func() { + <-building.Done() + if h.returned(r) { + h.said(r, link.KilledByHand, false) + } + h.end(r) + }() + said, err := h.kill("build-2") + if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") { + t.Fatalf("kill said %q (%v)", said, err) + } +} diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 5928a12..0d6bdf1 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -19,11 +19,13 @@ import ( "context" "encoding/json" "fmt" + "log" "net/url" "os" "os/signal" "strings" "syscall" + "time" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/builder" @@ -107,48 +109,96 @@ func run() error { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - machine, err := takeWorkFrom(credential, on) + js, seat, err := dialFor(credential) if err != nil { return err } + defer js.Close() + + // **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The + // paused flag is read from the workspace before anything is taken, so a holder restarted while + // paused takes nothing. + h := newHolder(on, seat, workspace, func(state link.HolderState) error { + body, err := json.Marshal(state) + if err != nil { + return err + } + _, err = js.Context().Publish(link.BuildPausedOf(seat, on), body) + return err + }) + if h.Paused() { + fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace)) + } + machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused}) defer machine.Close() + // The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired + // one serves none, and a subscription its holder has no grant for would be refused for ever. + if seat == link.TheBuildMachine { + stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(), + log.New(os.Stderr, "", 0)) + if err != nil { + return err + } + defer stopServing() + // And says so, for the console to find (novox/hq ADR 0197). + stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce( + announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0)) + if err != nil { + return err + } + defer stopAnnouncing() + go h.stateWhilePaused(ctx, time.Hour) + } + fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry) publisher := builder.Registry{Address: registry, Run: builder.Command} return machine.Take(ctx, func(ctx context.Context, work link.Build) { - answer(ctx, publisher, on, workspace, work) + answer(ctx, publisher, on, workspace, work, h) }) } -// takeWorkFrom opens this machine's link to whichever bus the mesh is on. +// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it +// holds. // // **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build // machine told about both would take work from one and answer on the other, and every log line would // say it was fine. -func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) { +func dialFor(credential Credential) (*broker.JetStream, string, error) { // **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5). // A credential for the mesh's bus carries user, password and fingerprint beside the address, // and that is enough to dial it, pinned. if !credential.onTheNewBus() { - return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL) + return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL) } js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint) if err != nil { - return nil, err + return nil, "", err } // **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the // handover): the mesh issues a build machine's credential naming the seat its module claims, // and one binary serves the old role as `builder` and the new as `build-agent` from that alone. seat := link.BuildSeatClaimed(credential.seatsClaimed()) fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat) - return link.MachineOverNATSOn(js, on, seat), nil + return js, seat, nil } // answer does one build and says what happened, whichever way it went. -func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) { +func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) { request := work.Request() + // **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends + // this build's commands and nothing else; the machine's own context ending — a SIGTERM — still + // reaches it through the parent, and that keeps today's meaning below. + building, cancel := context.WithCancel(ctx) + defer cancel() + var mine *running + if h != nil { + mine = h.begin(request, cancel) + defer h.end(mine) + } + // **First thing, and to stdout.** A build request that arrives and produces no visible line until // it either finishes or fails is indistinguishable from one that never arrived — which cost a long // diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that @@ -162,6 +212,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri say := func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) work.Say(step, message) + if mine != nil && step != "run" && step != "output" { + h.stepped(mine, step) + } } builder.Said = say defer func() { builder.Said = nil }() @@ -188,11 +241,24 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri // The package-registry credential is a build input, so it is resolved before the clone: a // build that could not have resolved its dependencies is refused in front of the reason, not // after a clone that then fails at npm ci. - built, err = builder.Build(ctx, builder.Command, publisher, + // Every container it starts is labelled with its id, so a kill finds what outlived the + // docker client (ADR 0219). + built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher, request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, forgeFrom(), say, request.Seats) } - if err != nil { + // Only a build the kill ended: the kill came before its work did. One that finished — built, or + // failed on its own — in the moment the kill arrived says what it did, and the kill is refused. + killed := false + if mine != nil { + killed = h.returned(mine) && err != nil + } + if killed { + // **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the + // error it ended with is the kill's consequence, not a fault of the source. + result.Failed = link.KilledByHand + say("failed", link.KilledByHand) + } else if err != nil { // A failure is a result. A build that fails and says nothing is indistinguishable from a // builder that is not running, and those want completely different responses. result.Failed = err.Error() @@ -217,7 +283,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri } } - if err := work.Announce(ctx, result); err != nil { + // A killed build is announced on a context of its own: the build's was the one cancelled, and the + // outcome must go out and the ask be settled — acknowledged, never redelivered to another machine + // to be built again. A machine being stopped is the other case and keeps its meaning: the + // parent's context is gone, nothing is announced or settled, and the ask is redelivered. + announcing := ctx + if killed { + fresh, stop := context.WithTimeout(context.Background(), 30*time.Second) + defer stop() + announcing = fresh + } + announceErr := work.Announce(announcing, result) + if mine != nil { + h.said(mine, result.Failed, announceErr == nil) + } + if err := announceErr; err != nil { // Said, not fatal: the build happened. A build reported as failed because announcing it // failed is a lie about work that was done — and the request stays unsettled below only if // nothing was said at all, so another machine can try. diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 6172e39..915fcfc 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -392,22 +392,34 @@ func buildBehind(ctx context.Context, wait time.Duration) error { // // Separated from the command so `--behind` can walk a list without a second path to the same act. func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error { + _, err := buildOneAsked(ctx, source, path, ref, wait, false) + return err +} + +// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome +// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked +// at, never taken in (issue 240), which is what `replay` asks unless told to register. +func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration, + dryRun bool) (string, error) { + if dryRun && wait != 0 { + return "", errors.New("a dry run waited for is `build --dry-run`") + } // Before anything is asked of a builder: a source on a seat nobody holds is refused here, with // the reason, rather than sent to a machine to fail at `git clone`. repository, err := cloneFrom(ctx, source) if err != nil { - return err + return "", err } ident, err := openIdentity(ctx) if err != nil { - return err + return "", err } defer ident.Close() server, err := connectLink(ctx, nil, nil, nil) if err != nil { - return err + return "", err } defer server.Close() @@ -420,6 +432,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti Ref: ref, Held: heldBy(ctx), Seats: seatBases(ctx), + DryRun: dryRun, } fmt.Printf("asked for %s", source) if source.Seat != "" { @@ -438,7 +451,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti seat := buildSeatHeld(ctx) ask, err := askOverOn(seat) if err != nil { - return err + return "", err } defer ask.Close() fmt.Printf(" of %s\n", seat) @@ -449,26 +462,31 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti // is still here. A tool call cannot hold a connection for the minutes a build takes; it // follows the build by its id instead. if err := ask.Ask(ctx, request); err != nil { - return err + return "", err + } + if dryRun { + fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+ + "its outcome is not taken in\n", request.ID) + return request.ID, nil } fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+ "shows what came of it; the module is registered when the outcome comes\n", request.ID) - return nil + return request.ID, nil } result, err := ask.Submit(ctx, request, wait) if err != nil { - return err + return request.ID, err } open, err := openStores(ctx) if err != nil { - return err + return request.ID, err } defer open.Close() manifest, kept, err := takeIn(ctx, open.inventory, result) if err != nil { - return err + return request.ID, err } // Said as recorded: what each artifact is, not where this builder happened to push it. for _, made := range kept.Made { @@ -478,7 +496,7 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti manifest.Module, manifest.Version, result.On, short(result.Commit)) saysWhenThePolicyActs(ctx, open.inventory, manifest.Module) fmt.Printf(" run `assign %s` to put it somewhere\n", manifest.Module) - return nil + return request.ID, nil } // saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the @@ -628,6 +646,8 @@ type answers struct { reported []inventory.Reported // plans is what the last merges produced and where each stands (novox/hq ADR 0162). plans []inventory.Plan + // paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219). + paused pauseView // refused is why a machine cannot be worked out at all, by name. A different thing from every // other answer here: those are about a machine that was told something, and this is about one // that cannot be told anything — it never reaches waiting, because nothing was computed for it diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index b15f40d..38fe897 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -73,6 +73,21 @@ func run() error { return askCommand(ctx, args[1:]) case "builds": return buildsCommand(ctx, args[1:]) + // The build queue, controlled by hand (novox/hq ADR 0219). + case "queue": + return queueCommand(ctx, args[1:]) + case "cancel": + return cancelCommand(ctx, args[1:]) + case "clear": + return clearCommand(ctx, args[1:]) + case "rebuild": + return rebuildCommand(ctx, args[1:]) + case "replay": + return replayCommand(ctx, args[1:]) + case "kill": + return killCommand(ctx, args[1:]) + case "pause", "resume": + return pauseCommand(ctx, args[0], args[1:]) case "collection": return collectionCommand(ctx, args[1:]) case "plans": @@ -202,6 +217,14 @@ func usage() { build --behind build every module the mesh holds older than its source build --on rebuild every module that stands on this module's artifacts, bases first builds [] what has been built lately, and what came of it + queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead + cancel drop a waiting or dead ask; recorded failed, cancelled by hand + clear [--dead] cancel every waiting ask (and the dead ones); never one in flight + rebuild ask the module's source again, or that build's, under a new id + replay [--register [--older]] that build's commit again; a dry run unless --register + kill end a build where it runs; recorded failed, killed by hand + pause [] / resume [] the build seat's holder there, or every holder, takes nothing new / again + plans retry ask a failed plan's failed builds again, and carry the plan on collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go builder issue a broker account for a build machine, scoped to build work, delivered as the builder module's broker secret (module add it first) @@ -271,7 +294,7 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error { case err != nil && result.Failed != "": fmt.Printf("%s: %v\n", result.ID, err) if result.Module != "" { - planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked) + planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID) } else { planFailedBuild(ctx, b.open, result) } @@ -280,18 +303,18 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error { // Not a failure: the module is already at what a later request built. A plan that asked // before that later request is answered by it; one that asked after it ignores this. fmt.Printf("%s: %v\n", result.ID, err) - planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked) + planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID) return nil case err != nil: fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err) if manifest.Module != "" { - planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked) + planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID) } return nil } fmt.Printf("%s: %s %s registered, built on %s from %s\n", result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit)) saysWhenThePolicyActs(ctx, b.inv, manifest.Module) - planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked) + planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID) return nil } diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go new file mode 100644 index 0000000..4bbf851 --- /dev/null +++ b/cmd/mesh-controller/plan_retry.go @@ -0,0 +1,401 @@ +package main + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A plan follows what is done to the build queue (novox/hq ADR 0219). +// +// Two things a person does to builds by hand would otherwise leave a plan saying something untrue: +// +// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder +// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE +// sends a reader looking for a fault that is a decision. It says what it waits on instead. +// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a +// failed plan's failed modules and the plan goes on from that tier as if they had built the +// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running +// beside it — beside it, the plan would either ask it again or stay failed on an outcome the +// rebuild has already replaced. + +// pauseView is whether the build seat takes work: the holders that said they are paused, and +// whether that is every holder. +type pauseView struct { + Nodes []string + All bool +} + +// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan +// building, whose current tier has an ask outstanding, while every holder of the seat is paused. +func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) { + if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) { + return "", false + } + var asked *time.Time + for _, m := range p.Tiers[p.Tier] { + if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil { + if asked == nil || s.AskedAt.Before(*asked) { + asked = s.AskedAt + } + } + } + if asked == nil { + return "", false + } + waited := now.Sub(*asked) + said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)", + strings.Join(pause.Nodes, ", "), waited.Round(time.Second)) + // Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held + // longer than a day is named. + if waited > pausedTooLong { + said += " — PAUSED OVER A DAY: `resume` takes builds again" + } + return said, true +} + +// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long. +const pausedTooLong = 24 * time.Hour + +// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case +// where the seat being paused changes what a plan says. +func awaitsABuild(plans []inventory.Plan) bool { + for _, p := range plans { + if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) { + continue + } + for _, m := range p.Tiers[p.Tier] { + if s := p.Modules[m]; s != nil && s.State == "asked" { + return true + } + } + } + return false +} + +// buildSeatPause reads whether the build seat's holders take work, from what each last said on the +// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building +// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then +// reads as late, which is what it said before this existed. +func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView { + if !awaitsABuild(plans) { + return pauseView{} + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return pauseView{} + } + seat := buildSeatAmong(entries) + holders := holdersAmong(entries, seat) + if len(holders) == 0 { + return pauseView{} + } + address, err := broker.BusAddress() + if err != nil { + return pauseView{} + } + js, err := broker.Dial(address) + if err != nil { + fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err) + return pauseView{} + } + defer js.Close() + said, err := link.PausedSaid(js, seat, holders) + if err != nil { + fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err) + return pauseView{} + } + return pauseOf(holders, said) +} + +// pauseOf is the view from what each holder said. +func pauseOf(holders []string, said map[string]link.HolderState) pauseView { + var v pauseView + for _, n := range holders { + if said[n].Paused { + v.Nodes = append(v.Nodes, n) + } + } + sort.Strings(v.Nodes) + v.All = len(holders) > 0 && len(v.Nodes) == len(holders) + return v +} + +// failedIn is the modules of a plan's current tier that failed to build, sorted. +func failedIn(p inventory.Plan) []string { + if p.Tier >= len(p.Tiers) { + return nil + } + var out []string + for _, m := range p.Tiers[p.Tier] { + if s := p.Modules[m]; s != nil && s.State == "failed" { + out = append(out, m) + } + } + sort.Strings(out) + return out +} + +// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan +// that holds what this one held now (issue 254, ADR 0218). +func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) { + for _, q := range plans { + if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) || + (p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) { + continue + } + return q, true + } + return inventory.Plan{}, false +} + +// retryRefusal is why a plan cannot be retried, or nothing. +func retryRefusal(p inventory.Plan, plans []inventory.Plan) error { + switch { + case p.State == inventory.PlanDone: + return fmt.Errorf("%s is done; there is nothing to retry", p.ID) + case p.State == inventory.PlanSuperseded: + return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note) + case p.Open(): + return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild ` asks one module again", p.ID, p.State) + } + if len(failedIn(p)) > 0 { + if q, found := newerOpenPlan(p, plans); found { + return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+ + "what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID) + } + return nil + } + stopped := stoppedRollouts(p) + if len(stopped) == 0 { + return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s", + p.Tier, p.ID, p.Note) + } + // **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or + // sent — a newer build, and sending this one again would put the older build back on its machines. + for _, m := range stopped { + if q, found := newerPlanFor(m, p, plans); found { + return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+ + "the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID) + } + } + return nil +} + +// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine +// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept. +func stoppedRollouts(p inventory.Plan) []string { + if p.Tier >= len(p.Tiers) { + return nil + } + var out []string + for _, m := range p.Tiers[p.Tier] { + if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil && + len(s.First) > 0 && s.Why != "" { + out = append(out, m) + } + } + sort.Strings(out) + return out +} + +// newerPlanFor is a plan made after this one that holds the module, superseded ones aside. +func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) { + for _, q := range plans { + if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) { + continue + } + for _, tier := range q.Tiers { + for _, m := range tier { + if m == module { + return q, true + } + } + } + } + return inventory.Plan{}, false +} + +// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A +// variable so a test of a retried rollout needs no machine. +var sendRollout = sendToEach + +// resumed sets a failed plan building again once nothing in its tier is failed. +func resumed(p *inventory.Plan, why string) { + if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 { + p.State = inventory.PlanBuilding + p.Note = why + } +} + +// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at +// that tier: what follows is the plan going on as if they had built the first time. +func retryPlan(ctx context.Context, open *stores, id string) (string, error) { + inv := open.inventory + release, err := inv.HoldPlans(ctx, true) + if err != nil { + return "", err + } + defer release() + p, err := inv.PlanByID(ctx, id) + if err != nil { + return "", err + } + plans, err := inv.OpenPlans(ctx) + if err != nil { + return "", err + } + recent, err := inv.RecentPlans(ctx, 50) + if err != nil { + return "", err + } + plans = append(plans, recent...) + if err := retryRefusal(p, plans); err != nil { + return "", err + } + if len(failedIn(p)) == 0 { + return retryRollouts(ctx, open, &p) + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return "", err + } + byName := map[string]inventory.Entry{} + for _, e := range entries { + byName[e.Manifest.Module] = e + } + failed := failedIn(p) + var asked []string + for _, m := range failed { + askModule(ctx, &p, m, byName) + if s := p.Modules[m]; s.State == "asked" { + asked = append(asked, m+" as "+s.Build) + } + } + resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", "))) + if err := inv.SavePlan(ctx, p); err != nil { + return "", err + } + if p.State != inventory.PlanBuilding { + return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note) + } + return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does", + p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil +} + +// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans +// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one. +func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) { + inv := open.inventory + release, err := inv.HoldPlans(ctx, true) + if err != nil { + return false, "", err + } + defer release() + openPlans, err := inv.OpenPlans(ctx) + if err != nil { + return false, "", err + } + recent, err := inv.RecentPlans(ctx, 20) + if err != nil { + return false, "", err + } + p, found := planHolding(module, openPlans, recent) + if !found { + return false, "", nil + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return false, "", err + } + byName := map[string]inventory.Entry{} + for _, e := range entries { + byName[e.Manifest.Module] = e + } + was := p.State + askModule(ctx, &p, module, byName) + s := p.Modules[module] + resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module)) + if err := inv.SavePlan(ctx, p); err != nil { + return false, "", err + } + if s.State != "asked" { + return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why) + } + said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome", + s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module) + switch { + case was == inventory.PlanFailed && p.State == inventory.PlanBuilding: + said += "; the plan had failed and builds again from this tier" + case was == inventory.PlanFailed: + said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them" + } + return true, said, nil +} + +// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not +// yet built, else the newest failed plan whose current tier does, and that no open plan of its +// repository supersedes. +func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) { + holds := func(p inventory.Plan) bool { + if p.Tier >= len(p.Tiers) { + return false + } + for _, m := range p.Tiers[p.Tier] { + if m == module { + s := p.Modules[m] + return s == nil || s.State != "built" + } + } + return false + } + for _, p := range openPlans { + if holds(p) { + return p, true + } + } + sorted := append([]inventory.Plan(nil), recent...) + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) }) + for _, p := range sorted { + if p.State != inventory.PlanFailed || !holds(p) { + continue + } + if _, superseded := newerOpenPlan(p, openPlans); superseded { + continue + } + return p, true + } + return inventory.Plan{}, false +} + +// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again, +// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan +// would have — the rest sent once those report they applied it, the next tier after (ADR 0218). +func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) { + var said []string + for _, m := range stoppedRollouts(*p) { + s := p.Modules[m] + sent, err := sendRollout(ctx, open, s.First) + if err != nil { + return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w", + m, strings.Join(s.First, ", "), p.ID, err) + } + now := time.Now().UTC() + s.First, s.FirstAt, s.Why = sent, &now, "" + said = append(said, m+" to "+strings.Join(sent, ", ")) + } + p.State = inventory.PlanRolling + p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; ")) + if err := open.inventory.SavePlan(ctx, *p); err != nil { + return "", err + } + return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+ + "applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 9a2d3c5..3e43f0a 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -1142,6 +1142,11 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string) if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil { return err } + // And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask + // whether it was cancelled the moment it took it. + if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil { + return err + } // Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from // registration, so a module reading one may watch it before its owner runs anywhere. One that // nothing declares any more is said and kept — what it holds is data. diff --git a/cmd/mesh-controller/queue.go b/cmd/mesh-controller/queue.go new file mode 100644 index 0000000..949ec9f --- /dev/null +++ b/cmd/mesh-controller/queue.go @@ -0,0 +1,709 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The build queue, controlled by hand (novox/hq ADR 0219). +// +// Seen whole — what waits, what runs where and for how long, what was handed out as often as it +// may be and never settled — and changed through the controller: an ask cancelled, the queue +// cleared, a module rebuilt, a build replayed. What one machine is running is that machine's +// holder's to end or pause, and the controller asks it (`kill`, `pause`, `resume`). +// +// **Everything that drops an ask leaves a failed outcome for it**, taken in exactly as a build that +// failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so, +// rather than waiting for ever on an answer nobody will give. + +// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer +// than the holder's worst case (its two passes removing containers and its wait between, 50s). +const ( + holderAsks = 15 * time.Second + killAnswer = 75 * time.Second +) + +// dialTheBus opens the controller's own connection, for a command that reads or changes the queue. +func dialTheBus() (*broker.JetStream, error) { + address, err := broker.BusAddress() + if err != nil { + return nil, err + } + js, err := broker.Dial(address) + if err != nil { + return nil, fmt.Errorf("cannot reach the bus: %w", err) + } + return js, nil +} + +// queueCommand prints every ask in the build seat's work queue. +func queueCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("queue", flag.ContinueOnError) + asJSON := set.Bool("json", false, "the queue as JSON") + if _, err := parseAround(set, args); err != nil { + return err + } + js, err := dialTheBus() + if err != nil { + return err + } + defer js.Close() + seat := buildSeatHeld(ctx) + q, err := link.ReadQueue(ctx, js, seat) + if err != nil { + return err + } + if *asJSON { + body, err := json.MarshalIndent(q, "", " ") + if err != nil { + return err + } + fmt.Println(string(body)) + return nil + } + fmt.Print(queueText(q, time.Now())) + return nil +} + +// queueText is the queue as a person reads it: a summary line, then each kind in queue order. +// Never what an ask carries as `held` — that is every artifact the mesh has built. +func queueText(q link.Queue, now time.Time) string { + var b strings.Builder + waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead) + fmt.Fprintf(&b, "%s: %d waiting, %d in flight, %d dead\n", q.Seat, len(waiting), len(running), len(dead)) + ago := func(t time.Time) string { + if t.IsZero() { + return "asked at an unknown time" + } + return "asked " + now.Sub(t).Round(time.Second).String() + " ago" + } + what := func(a link.QueuedAsk) string { + s := a.Repository + if a.Path != "" { + s += " at " + a.Path + } + if a.Ref != "" { + s += " on " + a.Ref + } + return s + } + if len(running) > 0 { + fmt.Fprintln(&b, "\nin flight:") + for _, a := range running { + where := "taken, not yet said where" + switch { + case a.On != "": + where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second)) + case a.Was != "": + where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was) + } + fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq) + } + } + if len(waiting) > 0 { + fmt.Fprintln(&b, "\nwaiting:") + for _, a := range waiting { + fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq) + } + } + if len(dead) > 0 { + fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver) + for _, a := range dead { + fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq) + } + } + switch { + case len(q.Asks) == 0: + fmt.Fprintln(&b, "nothing is asked of it") + default: + fmt.Fprintln(&b, "\n`cancel ` drops a waiting or dead ask, `clear` every waiting one, `kill ` ends one in flight") + } + return b.String() +} + +// cancelCommand drops one waiting or dead ask. +func cancelCommand(ctx context.Context, args []string) error { + if len(args) != 1 { + return errors.New("cancel ") + } + js, err := dialTheBus() + if err != nil { + return err + } + defer js.Close() + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + seat := buildSeatHeld(ctx) + q, err := link.ReadQueue(ctx, js, seat) + if err != nil { + return err + } + ask, found := q.Find(args[0]) + if !found { + return fmt.Errorf("%s is not in the %s queue: it was built, cancelled, or never asked — `builds` says "+ + "what came of it", args[0], seat) + } + said, err := cancelAsk(ctx, js, open, seat, ask) + if err != nil { + return err + } + fmt.Println(said) + return nil +} + +// cancelAsk drops one ask from the queue and records it failed, cancelled by hand. +// +// **In this order, and each step for a reason** (novox/hq ADR 0219): +// 1. an ask a machine says it is building is refused: deleting its message ends nothing a machine +// is running — that is `kill`, on the machine running it; +// 2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it; +// 3. for a waiting ask, the worker is read again: one handed out since the queue was read was +// taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either +// read the mark first and ends it as cancelled, or is building it; +// 4. for an ask the worker counts handed out that no machine is building — taken and not yet said, +// handed back after a restart, or past its deliveries while nobody pulls — the holder's own look +// at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder +// that took it before the mark is building it, and only `kill` ends that; +// 5. the message is deleted by its sequence; +// 6. the failed outcome is taken in as any failed build's is, and the plan that asked fails. +func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) { + if ask.State == link.AskInFlight && ask.On != "" { + return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+ + "ends the build where it runs", ask.ID, ask.On, ask.ID) + } + if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil { + return "", err + } + withdraw := func() { + if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil { + fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err) + } + } + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}}) + switch ask.State { + case link.AskWaiting: + if taken, err := takenSince(js, worker, ask.Seq); err != nil { + withdraw() + return "", err + } else if taken { + withdraw() + return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+ + "holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+ + "`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID) + } + case link.AskInFlight: + if started, err := startedSince(ctx, js, seat, ask); err != nil { + withdraw() + return "", err + } else if started != "" { + withdraw() + return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+ + "ends it there", ask.ID, started, ask.ID) + } + } + if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) && + !errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") { + return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+ + "holder taking it ends it as cancelled", ask.ID, ask.Seq, err) + } + recordCancelled(ctx, open, ask) + return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+ + "that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil +} + +// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it +// started: longer than a holder's look at the cancelled set (3s) and its start that follows. +var holderLooks = 5 * time.Second + +// startedSince waits out a holder's look at the cancelled set and says the machine that started the +// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not +// a start of a build. +func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(holderLooks): + } + since := time.Now().Add(-7 * 24 * time.Hour) + if !ask.AskedAt.IsZero() { + since = ask.AskedAt.Add(-time.Minute) + } + heard, err := link.ReadBuildEvents(ctx, js, seat, since) + if err != nil { + return "", err + } + s, ok := heard.Started[ask.ID] + if !ok || heard.Outcomes[ask.ID] { + return "", nil + } + at, _ := time.Parse(time.RFC3339Nano, s.At) + if ask.Started.IsZero() || at.After(ask.Started) { + return s.On, nil + } + return "", nil +} + +// takenSince is whether the worker has handed out the ask at this sequence. +func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) { + info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name) + if errors.Is(err, nats.ErrConsumerNotFound) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("cannot read the worker of the queue again: %w", err) + } + return info.Delivered.Stream >= seq, nil +} + +// recordCancelled takes the failed outcome in the way the daemon takes in any build's: recorded, +// then the plan that asked for it — by the id it asked with, or by repository and path. +func recordCancelled(ctx context.Context, open *stores, ask link.QueuedAsk) { + r := ask.Request + result := link.BuildResult{ID: ask.ID, Repository: ask.Repository, Path: ask.Path, Ref: ask.Ref, + Source: r.Source, DryRun: r.DryRun, Failed: link.CancelledByHand} + _ = builds{open.inventory, open}.Built(ctx, result) +} + +// clearCommand cancels every waiting ask, and with --dead every dead one too. +func clearCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("clear", flag.ContinueOnError) + dead := set.Bool("dead", false, "the dead asks too") + if _, err := parseAround(set, args); err != nil { + return err + } + js, err := dialTheBus() + if err != nil { + return err + } + defer js.Close() + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + seat := buildSeatHeld(ctx) + said, err := clearQueue(ctx, js, open, seat, *dead) + fmt.Print(said) + return err +} + +// clearQueue cancels what clear names, each as `cancel` would, and never anything in flight. +func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat string, dead bool) (string, error) { + q, err := link.ReadQueue(ctx, js, seat) + if err != nil { + return "", err + } + var b strings.Builder + cancelled, refused := 0, 0 + for _, a := range q.Asks { + if a.State == link.AskInFlight || (a.State == link.AskDead && !dead) { + continue + } + said, err := cancelAsk(ctx, js, open, seat, a) + if err != nil { + refused++ + fmt.Fprintf(&b, " %s: %v\n", a.ID, err) + continue + } + cancelled++ + fmt.Fprintf(&b, " %s\n", said) + } + what := "waiting" + if dead { + what = "waiting and dead" + } + fmt.Fprintf(&b, "%d %s ask(s) cancelled", cancelled, what) + if refused > 0 { + fmt.Fprintf(&b, ", %d could not be", refused) + } + fmt.Fprintln(&b) + if n := len(q.Of(link.AskInFlight)); n > 0 { + fmt.Fprintf(&b, "%d in flight, left running: `kill ` ends one where it runs\n", n) + } + if n := len(q.Of(link.AskDead)); n > 0 && !dead { + fmt.Fprintf(&b, "%d dead, left: `clear --dead` cancels them too\n", n) + } + if refused > 0 { + return b.String(), fmt.Errorf("%d ask(s) could not be cancelled", refused) + } + return b.String(), nil +} + +// rebuildCommand asks the module's current source again — or, given a build's id, that build's +// repository, path and ref — under a new id. +func rebuildCommand(ctx context.Context, args []string) error { + if len(args) != 1 { + return errors.New("rebuild ") + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + entries, err := inv.Catalogued(ctx) + if err != nil { + return err + } + var source buildSource + var path, ref, module string + if b, found, err := inv.BuildByID(ctx, args[0]); err != nil { + return err + } else if found { + source, module = sourceOfBuild(b, entries) + path, ref = b.Path, b.Ref + // **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue + // 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll + // that commit out over everything since. Building that commit again is `replay`, which says + // what it would do and refuses to register it while anything newer is asked or registered. + if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" { + ref = followedBranch(e.Source.Ref) + follows := ref + if follows == "" { + follows = "the repository's default branch" + } + fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+ + "builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID) + } + } else if e, known := entryNamed(entries, args[0]); known { + // As a plan asks it: the branch it follows, never a commit a build once named (issue 215). + source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat} + path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module + } else { + return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0]) + } + + // **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the + // plan would ask it again, or stay failed on an outcome a rebuild has replaced. + if module != "" { + joined, said, err := joinAPlan(ctx, open, module) + if err != nil { + return err + } + if joined { + fmt.Println(said) + return nil + } + } + id, err := askABuild(ctx, source, path, ref) + if err != nil { + return err + } + fmt.Printf("rebuild asked as %s\n", id) + return nil +} + +// entryNamed is the catalogue's entry for a module. +func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) { + for _, e := range entries { + if e.Manifest.Module == name { + return e, true + } + } + return inventory.Entry{}, false +} + +// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own +// source when the build is of one — on its seat, so the outcome registers it as the mesh records it +// (ADR 0111) — else the repository as it was cloned. +func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, string) { + for _, e := range entries { + if (b.Module != "" && e.Manifest.Module == b.Module) || + (b.Module == "" && repositoryMatches(e.Source.Repository, b.Repository) && e.Source.Path == b.Path) { + return buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}, e.Manifest.Module + } + } + return buildSource{Repository: b.Repository}, b.Module +} + +// replayCommand asks a recorded build's repository and path again at the commit it built. +func replayCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("replay", flag.ContinueOnError) + register := set.Bool("register", false, "register what it builds, as any build is") + older := set.Bool("older", false, "with --register: even though a newer build of the module is registered") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 1 { + return errors.New("replay [--register [--older]]") + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + b, found, err := inv.BuildByID(ctx, positionals[0]) + if err != nil { + return err + } + if !found { + return fmt.Errorf("no build %s is recorded", positionals[0]) + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return err + } + source, module := sourceOfBuild(b, entries) + var history []inventory.Build + if module != "" { + if history, err = inv.Builds(ctx, module, 100); err != nil { + return err + } + } + // What is asked of the module and not yet answered, when the replay would be registered. + var outstanding []string + if *register { + js, err := dialTheBus() + if err != nil { + return err + } + q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx)) + js.Close() + if err != nil { + return err + } + plans, err := inv.OpenPlans(ctx) + if err != nil { + return err + } + outstanding = outstandingFor(module, b.Repository, b.Path, q, plans) + } + if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil { + return err + } + id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register) + if err != nil { + return err + } + fmt.Println(replaySaid(b, module, id, *register)) + return nil +} + +// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207). +// +// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it +// was asked, which is what orders builds of one module (issue 219). So a registered replay of an +// older commit is newer than everything since, and would roll that older commit out as the module's +// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless +// --register says otherwise, and --register is refused when a newer build of a different commit is +// registered, unless --older says that is the point. +// +// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the +// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks, +// and their builds — made from newer source — would be recorded and never registered (issue 219 +// orders by ask), the plan waiting on them sending the older commit instead. +func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool, + outstanding []string) error { + if b.Commit == "" { + return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+ + "nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID) + } + if older && !register { + return errors.New("--older only says what --register may do; a dry run registers nothing") + } + if register && len(outstanding) > 0 { + return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+ + "replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look", + orNone(module), strings.Join(outstanding, "; "), b.ID) + } + if !register || older { + return nil + } + for _, h := range history { + if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit { + continue + } + if h.AskedOrAt().After(b.AskedOrAt()) { + return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+ + "would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+ + "without --register looks at it; --register --older registers it anyway", + module, h.ID, short(h.Commit), short(b.Commit), module, b.ID) + } + } + return nil +} + +// replaySaid is what a replay prints once asked: what it builds, and what becomes of the outcome. +func replaySaid(b inventory.Build, module, id string, register bool) string { + what := b.Repository + if module != "" { + what = module + } + said := fmt.Sprintf("replaying %s (%s) at %s as %s", b.ID, what, short(b.Commit), id) + if !register { + return said + "\n a dry run: the outcome is looked at and not taken in — nothing is recorded or " + + "registered, and nothing is sent. `builds --log " + id + "` follows it; `--register` registers it" + } + return said + "\n registered when it is built, as the module's current version — newer than every build " + + "asked before now — and rolled out as its policy says. `builds --log " + id + "` follows it" +} + +// killCommand finds the machine running a build and asks its holder to end it. +func killCommand(ctx context.Context, args []string) error { + if len(args) != 1 { + return errors.New("kill ") + } + id := args[0] + js, err := dialTheBus() + if err != nil { + return err + } + defer js.Close() + seat := buildSeatHeld(ctx) + since := time.Now().Add(-7 * 24 * time.Hour) + if at, ok := link.BuildAskedAt(id); ok { + since = at.Add(-time.Minute) + } + heard, err := link.ReadBuildEvents(ctx, js, seat, since) + if err != nil { + return err + } + started, ok := heard.Started[id] + if !ok { + return fmt.Errorf("no machine has said it started %s: if it waits in the queue, `cancel %s` drops it", id, id) + } + if heard.Outcomes[id] { + return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On) + } + answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer) + if err != nil { + return err + } + return printHolderAnswer(started.On, answer) +} + +// pauseCommand asks one machine's holder, or every holder's, to pause or resume. +func pauseCommand(ctx context.Context, verb string, args []string) error { + if len(args) > 1 { + return fmt.Errorf("%s [node]", verb) + } + js, err := dialTheBus() + if err != nil { + return err + } + defer js.Close() + seat := buildSeatHeld(ctx) + nodes := args + if len(nodes) == 0 { + if nodes, err = buildSeatHolders(ctx, seat); err != nil { + return err + } + if len(nodes) == 0 { + return fmt.Errorf("nothing holds %s, so there is nothing to %s", seat, verb) + } + } + failed := 0 + for _, node := range nodes { + answer, err := link.AskSeatTool(ctx, js.Conn(), seat, verb, node, map[string]string{}, holderAsks) + if err != nil { + fmt.Printf("%s: %v\n", node, err) + failed++ + continue + } + if err := printHolderAnswer(node, answer); err != nil { + failed++ + } + } + if failed > 0 { + return fmt.Errorf("%d of %d machine(s) did not %s", failed, len(nodes), verb) + } + return nil +} + +// printHolderAnswer prints what a holder said, or its refusal as the command's failure. +func printHolderAnswer(node string, answer link.Answer) error { + if answer.Error != "" { + fmt.Printf("%s: %s\n", node, answer.Error) + return errors.New(answer.Error) + } + var said struct { + Said string `json:"said"` + } + if json.Unmarshal(answer.Result, &said) == nil && said.Said != "" { + fmt.Printf("%s: %s\n", node, said.Said) + return nil + } + fmt.Printf("%s: %s\n", node, string(answer.Result)) + return nil +} + +// buildSeatHolders is every machine an assigned module holding the build seat runs on. +func buildSeatHolders(ctx context.Context, seat string) ([]string, error) { + open, err := openStores(ctx) + if err != nil { + return nil, err + } + defer open.Close() + entries, err := open.inventory.Catalogued(ctx) + if err != nil { + return nil, err + } + return holdersAmong(entries, seat), nil +} + +// holdersAmong is the machines of every catalogued module claiming the seat, sorted, once each. +func holdersAmong(entries []inventory.Entry, seat string) []string { + seen := map[string]bool{} + var out []string + for _, e := range entries { + if !e.Manifest.ClaimsSeat(seat) { + continue + } + for _, n := range e.On { + if !seen[n] { + seen[n] = true + out = append(out, n) + } + } + } + sort.Strings(out) + return out +} + +// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue, +// by repository and path, and every open plan holding it not yet built. +func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string { + var out []string + for _, a := range q.Asks { + if repositoryMatches(a.Repository, repository) && a.Path == path { + out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State)) + } + } + if module == "" { + return out + } + for _, p := range plans { + if !p.Open() { + continue + } + for _, tier := range p.Tiers { + for _, m := range tier { + if m == module { + if st := p.Modules[m]; st == nil || st.State != "built" { + out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID)) + } + } + } + } + } + return out +} diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go new file mode 100644 index 0000000..d78bb67 --- /dev/null +++ b/cmd/mesh-controller/queue_test.go @@ -0,0 +1,772 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it. +// +// docker run -d --rm --name bq-nats -p 14294:4222 nats:2.10-alpine -js +// make postgres PG_PORT=55566 PG_CONTAINER=bq-pg +// MESH_TEST_NATS=nats://127.0.0.1:14294 \ +// MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \ +// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused' + +// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked. +func asksRecorded(t *testing.T) *[]string { + t.Helper() + var asked []string + was := askABuild + askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { + id := link.NewBuildID(time.Now().Add(time.Duration(len(asked)) * time.Millisecond)) + asked = append(asked, source.Repository+"#"+id) + return id, nil + } + t.Cleanup(func() { askABuild = was }) + return &asked +} + +// twoTiers registers two modules, b standing on a, each with a source a plan asks. +func twoTiers(t *testing.T, open *stores) { + t.Helper() + for _, name := range []string{"a", "b"} { + if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"}, + inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil { + t.Fatal(err) + } + } +} + +// A failed plan is retried: its failed module asked again under a new id, the plan building at that +// tier, and when that build comes in the plan goes on and asks its next tier. +func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", + Why: link.KilledByHand}}} + if err := open.inventory.SavePlan(ctx, failed); err != nil { + t.Fatal(err) + } + + said, err := retryPlan(ctx, open, failed.ID) + if err != nil { + t.Fatal(err) + } + p, err := open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + a := p.Modules["a"] + if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" { + t.Fatalf("after retry the plan is %s and a is %+v", p.State, a) + } + if !strings.Contains(said, a.Build) { + t.Errorf("retry does not say the new id: %q", said) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } + + // The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows. + planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1") + if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding { + t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note) + } + asking, _ := link.BuildAskedAt(a.Build) + planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build) + p, err = open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" { + t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"]) + } + if len(*asked) != 2 { + t.Fatalf("asked %v", *asked) + } +} + +// What retry refuses, and says why. +func TestRetryRefusesWhatItCannotResume(t *testing.T) { + at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + plan := func(id, state string, created time.Time) inventory.Plan { + return inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: id + "c0ffee", + Created: created, State: state, Tiers: [][]string{{"a"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}} + } + failed := plan("plan-1", inventory.PlanFailed, at) + for _, c := range []struct { + p inventory.Plan + others []inventory.Plan + says string + }{ + {plan("plan-d", inventory.PlanDone, at), nil, "is done"}, + {plan("plan-s", inventory.PlanSuperseded, at), nil, "was"}, + {plan("plan-o", inventory.PlanBuilding, at), nil, "still building"}, + {failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"}, + } { + err := retryRefusal(c.p, c.others) + if err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says) + } + } + stopped := failed + stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}} + stopped.Note = "a stopped at its first machine" + if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") { + t.Errorf("a plan with nothing failed to build was retried: %v", err) + } + // Another branch's newer plan, an older one, and a failed one do not supersede it. + other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour)) + other.Branch = "release" + if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)), + plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil { + t.Errorf("refused for a plan that does not supersede it: %v", err) + } +} + +// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone. +func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}} + if err := open.inventory.SavePlan(ctx, failed); err != nil { + t.Fatal(err) + } + if err := rebuildCommand(ctx, []string{"a"}); err != nil { + t.Fatal(err) + } + p, err := open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" { + t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"]) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } + // b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone. + if err := rebuildCommand(ctx, []string{"b"}); err != nil { + t.Fatal(err) + } + if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil { + t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"]) + } + if len(*asked) != 2 { + t.Fatalf("asked %v", *asked) + } +} + +// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module. +func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) { + at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed, + Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}} + newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour), + State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}} + if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found { + t.Fatal("joined a failed plan a newer open one supersedes") + } + if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" { + t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID) + } + built := failed + built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}} + if _, found := planHolding("a", nil, []inventory.Plan{built}); found { + t.Fatal("joined a plan that has the module built") + } +} + +// replay is a dry run unless registered, and registering an older commit than one registered since +// is refused unless --older says it is meant (novox/hq issue 207). +func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) { + asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked} + newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)} + history := []inventory.Build{newer, old} + + if err := replayRefusal(old, "a", history, false, false, nil); err != nil { + t.Errorf("a dry run was refused: %v", err) + } + err := replayRefusal(old, "a", history, true, false, nil) + if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") { + t.Errorf("registering an older commit than the one registered was not refused: %v", err) + } + if err := replayRefusal(old, "a", history, true, true, nil); err != nil { + t.Errorf("--register --older was refused: %v", err) + } + if err := replayRefusal(newer, "a", history, true, false, nil); err != nil { + t.Errorf("registering the newest build again was refused: %v", err) + } + // A newer build of the same commit, or one that failed, is not a newer version to roll back from. + same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)} + broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)} + if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil { + t.Errorf("refused for a newer build of the same commit or a failed one: %v", err) + } + if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil { + t.Error("a build that recorded no commit was replayed") + } + if err := replayRefusal(old, "a", history, false, true, nil); err == nil { + t.Error("--older without --register was taken") + } + // **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan + // holding it unbuilt, would be replaced by a replay asked now (issue 219). + q := link.Queue{Asks: []link.QueuedAsk{ + {ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting}, + {ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting}, + {ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting}, + }} + plans := []inventory.Plan{ + {ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}, + {ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}, + {ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}, + } + outstanding := outstandingFor("a", "novox/a", "", q, plans) + if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") { + t.Fatalf("outstanding: %v", outstanding) + } + if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") { + t.Errorf("registered over an outstanding ask: %v", err) + } + if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil { + t.Errorf("a dry run was refused for what is outstanding: %v", err) + } + if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") { + t.Errorf("a dry replay does not say what it is: %q", said) + } + if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") { + t.Errorf("a registered replay does not say what it does: %q", said) + } +} + +// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat +// paused on some holders only is not a reason the plan is waiting. +func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) { + now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) + asked := now.Add(-12 * time.Minute) + p := inventory.Plan{ID: "plan-p", Repository: "novox/a", Commit: "c0ffee", State: inventory.PlanBuilding, + Updated: now.Add(-2 * time.Hour), Tiers: [][]string{{"a"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}} + + all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}}) + line := planLineWith(p, now, all) + if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") { + t.Errorf("a plan on a paused seat reads %q", line) + } + if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") { + t.Errorf("status --json says %+v", st) + } + if _, late := openPlans([]inventory.Plan{p}, all); late != 0 { + t.Errorf("a plan waiting on a paused seat counted late") + } + + longAgo := now.Add(-25 * time.Hour) + forgotten := p + forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}} + if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") { + t.Errorf("a plan paused over a day reads %q", line) + } + + some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}}) + if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) { + t.Fatalf("%+v", some) + } + if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") { + t.Errorf("a seat paused on one holder of two made the plan not late: %q", line) + } + if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late { + t.Errorf("status --json: %+v", st) + } + // A plan rolling out, or with nothing asked, is not waiting on the seat. + rolling := p + rolling.State = inventory.PlanRolling + if _, paused := pausedWaiting(rolling, all, now); paused { + t.Error("a rolling plan reads as waiting on the build seat") + } +} + +// The queue's verbs are the controller seat's, each to the command it names. +func TestTheQueueVerbsRunTheirCommands(t *testing.T) { + for _, c := range []struct { + verb string + args map[string]any + want []string + }{ + {"queue", nil, []string{"queue"}}, + {"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}}, + {"clear", nil, []string{"clear"}}, + {"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}}, + {"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}}, + {"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}}, + {"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}}, + {"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}}, + {"pause", nil, []string{"pause"}}, + {"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}}, + {"plans", map[string]any{"retry": "plan-1"}, []string{"plans", "retry", "plan-1"}}, + } { + got, err := argvFor(c.verb, c.args) + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want) + } + } + if _, err := argvFor("cancel", nil); err == nil { + t.Error("cancel without an id was taken") + } + declared := map[string]bool{} + for _, v := range catalogue.ControllerVerbs { + declared[v.Name] = true + } + for _, v := range []string{"queue", "cancel", "clear", "rebuild", "replay", "kill", "pause", "resume"} { + if !declared[v] { + t.Errorf("%s is not a verb of the controller seat", v) + } + } +} + +// --- against a real bus ----------------------------------------------------------------------- + +// aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller +// pointed at it, and a function that asks the seat one build. +func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) { + t.Helper() + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + t.Setenv(broker.NATSVar, url) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}, + Emits: []string{"started", "built", "log.*", "paused.*"}} + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") + if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{ + link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil { + t.Fatal(err) + } + if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") + _ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine)) + _ = js.Context().PurgeStream(broker.EventsStream) + }) + ask := func(id, repository string) link.BuildRequest { + r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}} + body, _ := json.Marshal(r) + if _, err := js.Context().Publish(link.BuildWork(), body); err != nil { + t.Fatal(err) + } + return r + } + return js, ask +} + +// deadOne takes the oldest ask from the worker and hands it back as often as the worker allows. +func deadOne(t *testing.T, js *broker.JetStream) { + t.Helper() + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) + sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + for i := 0; i < worker.MaxDeliver; i++ { + msgs, err := sub.Fetch(1, nats.MaxWait(3*time.Second)) + if err != nil { + t.Fatalf("delivery %d: %v", i+1, err) + } + _ = msgs[0].Nak() + } + // One more pull, as a holder always has one waiting: the server finds the ask past its deliveries + // then, and stops counting it pending. + if msgs, _ := sub.Fetch(1, nats.MaxWait(time.Second)); len(msgs) > 0 { + t.Fatalf("an ask past its deliveries was delivered again") + } +} + +// cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan +// that asked for it, matched by the id, fails with it; an ask the plan's records name no module for +// is still found. +func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + twoTiers(t, open) + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + asked, _ := link.BuildAskedAt(id) + plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked, + State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}} + if err := open.inventory.SavePlan(ctx, plan); err != nil { + t.Fatal(err) + } + + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id { + t.Fatalf("the queue reads %+v", q) + } + if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") || + strings.Contains(text, "secret-ish") { + t.Errorf("the queue says:\n%s", text) + } + + if err := cancelCommand(ctx, []string{id}); err != nil { + t.Fatal(err) + } + if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { + t.Fatalf("the ask is still queued: %+v", q.Asks) + } + if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled { + t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err) + } + b, found, err := open.inventory.BuildByID(ctx, id) + if err != nil || !found || b.Failed != link.CancelledByHand { + t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err) + } + p, err := open.inventory.PlanByID(ctx, plan.ID) + if err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand { + t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"]) + } + if err := cancelCommand(ctx, []string{id}); err == nil { + t.Error("an ask cancelled already was cancelled again") + } +} + +// clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight. +func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + start := time.Now() + dead := link.NewBuildID(start) + ask(dead, "https://forge.example/novox/dead.git") + deadOne(t, js) + var waiting []string + for i := 1; i <= 2; i++ { + id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond)) + waiting = append(waiting, id) + ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i)) + } + + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 { + t.Fatalf("the queue reads %+v", q) + } + said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") { + t.Errorf("clear said:\n%s", said) + } + q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine) + if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead { + t.Fatalf("after clear the queue is %+v", q.Asks) + } + if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil { + t.Fatal(err) + } + if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { + t.Fatalf("clear --dead left %+v", q.Asks) + } + for _, id := range append(waiting, dead) { + if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand { + t.Errorf("%s is recorded as %+v", id, b) + } + } +} + +// An ask in flight is not cancelled: kill ends it where it runs. +func TestCancelRefusesAnAskInFlight(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) + sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { + t.Fatal(err) + } + started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { + t.Fatal(err) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + a, _ := q.Find(id) + if a.State != link.AskInFlight || a.On != "ace" { + t.Fatalf("the taken ask reads %+v", a) + } + _, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a) + if err == nil || !strings.Contains(err.Error(), "kill "+id) { + t.Fatalf("an ask in flight was cancelled: %v", err) + } + if _, found, _ := open.inventory.BuildByID(ctx, id); found { + t.Error("a refused cancel recorded an outcome") + } +} + +// kill finds the machine from the build's start and asks that machine's holder; pause asks the +// machine named. Each prints what the holder answered, and a refusal is the command's failure. +func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) { + js, _ := aBuildQueue(t) + ctx := t.Context() + asked := map[string]string{} + handlers := map[string]link.ToolHandler{ + "kill": func(_ context.Context, raw json.RawMessage) (any, error) { + var args struct{ ID string } + _ = json.Unmarshal(raw, &args) + asked["kill"] = args.ID + if args.ID != "build-running" { + return nil, fmt.Errorf("ace is not building %s", args.ID) + } + return map[string]any{"said": "killed " + args.ID}, nil + }, + "pause": func(context.Context, json.RawMessage) (any, error) { + asked["pause"] = "ace" + return map[string]any{"said": "ace is paused"}, nil + }, + } + stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil) + if err != nil { + t.Fatal(err) + } + defer stop() + for _, id := range []string{"build-running", "build-other"} { + started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { + t.Fatal(err) + } + } + if err := killCommand(ctx, []string{"build-running"}); err != nil { + t.Fatal(err) + } + if asked["kill"] != "build-running" { + t.Fatalf("the holder was asked %v", asked) + } + if err := killCommand(ctx, []string{"build-other"}); err == nil { + t.Error("the holder's refusal was not the command's") + } + if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") { + t.Errorf("a build nobody started: %v", err) + } + if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" { + t.Fatalf("pause: %v %v", err, asked) + } + if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil { + t.Error("a machine nothing answers on was resumed") + } +} + +// A plan that stopped at its first machine is retried: the module sent to that machine again, the +// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module. +func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + var sentTo [][]string + was := sendRollout + sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { + sentTo = append(sentTo, names) + return names, nil + } + t.Cleanup(func() { sendRollout = was }) + + long := time.Now().UTC().Add(-2 * time.Hour) + stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a stopped at its first machine in tier 0: laptop refused what it was sent", + Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee", + First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}} + if err := open.inventory.SavePlan(ctx, stopped); err != nil { + t.Fatal(err) + } + + // A newer plan holding a refuses it: sending the older build would put it back. + newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour), + State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}} + if err := open.inventory.SavePlan(ctx, newer); err != nil { + t.Fatal(err) + } + if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") { + t.Fatalf("retried under a newer plan: %v", err) + } + newer.State = inventory.PlanSuperseded + if err := open.inventory.SavePlan(ctx, newer); err != nil { + t.Fatal(err) + } + + said, err := retryPlan(ctx, open, stopped.ID) + if err != nil { + t.Fatal(err) + } + if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") { + t.Fatalf("sent %v; said %q", sentTo, said) + } + p, err := open.inventory.PlanByID(ctx, stopped.ID) + if err != nil { + t.Fatal(err) + } + a := p.Modules["a"] + if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" { + t.Fatalf("after retry the plan is %s, a %+v", p.State, a) + } + // And it goes on: a records (its policy sends nothing more), so the next tier is asked. + advancePlans(ctx, open) + if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" { + t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"]) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } +} + +// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside +// the plan, asked later, never answers it (novox/hq ADR 0219). +func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := time.Now().UTC().Add(-time.Minute) + plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked, + State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}} + if err := open.inventory.SavePlan(ctx, plan); err != nil { + t.Fatal(err) + } + planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay") + p, _ := open.inventory.PlanByID(ctx, plan.ID) + if p.Modules["a"].State != "asked" { + t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"]) + } + // From the records too: a later build of the module recorded is not the plan's. + recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}} + if settleFromRecords(&p, p.Tiers[0], recorded, nil) { + t.Fatalf("the records settled it with another build: %+v", p.Modules["a"]) + } + planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own") + if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" { + t.Fatalf("its own build did not settle it: %+v", p.Modules["a"]) + } +} + +// rebuild of a build made at a commit asks what the module follows now, never the commit. +func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + var refs []string + was := askABuild + askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) { + refs = append(refs, ref) + return was(c, source, path, ref) + } + if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a", + Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil { + t.Fatal(err) + } + if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil { + t.Fatal(err) + } + if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 { + t.Fatalf("asked %v at %v", *asked, refs) + } +} + +// An ask the worker counts out that no machine said it started is cancelled only if no start comes +// while a holder looks: one that does withdraws the cancel, and kill is what ends it. +func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + was := holderLooks + holderLooks = 300 * time.Millisecond + t.Cleanup(func() { holderLooks = was }) + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) + sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { + t.Fatal(err) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + a, _ := q.Find(id) + if a.State != link.AskInFlight || a.On != "" { + t.Fatalf("the taken ask reads %+v", a) + } + // The holder that took it says it started, while the cancel waits. + go func() { + time.Sleep(100 * time.Millisecond) + started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + _, _ = js.Context().Publish(link.BuildStarted(), started) + }() + if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") { + t.Fatalf("a build that started was cancelled: %v", err) + } + if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled { + t.Error("the withdrawn cancel is still marked") + } + if _, found, _ := open.inventory.BuildByID(ctx, id); found { + t.Error("a withdrawn cancel recorded an outcome") + } +} diff --git a/cmd/mesh-controller/readable.go b/cmd/mesh-controller/readable.go index 6049fd0..119f9f7 100644 --- a/cmd/mesh-controller/readable.go +++ b/cmd/mesh-controller/readable.go @@ -176,7 +176,7 @@ func statusAsJSON(asked answers) ([]byte, error) { out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{}, Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{}, Reported: []machineReported{}, Unresolved: []machineUnresolved{}, - Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now())} + Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)} // In a stated order, so two readings of an unchanged mesh are the same document. untakenNodes := make([]string, 0, len(asked.untaken)) for name := range asked.untaken { diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 499760a..0f527fa 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -324,37 +324,52 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e for _, e := range entries { byName[e.Manifest.Module] = e } - now := time.Now().UTC() for _, name := range p.Tiers[p.Tier] { - state := p.Modules[name] - if state == nil { - state = &inventory.PlanModule{} - p.Modules[name] = state - } - e, known := byName[name] - if !known { - state.State = "failed" - state.Why = "no longer in the catalogue" - p.State = inventory.PlanFailed - p.Note = name + " is no longer in the catalogue" - continue - } - source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat} - fmt.Printf(" tier %d: ", p.Tier) - // The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215). - if err := buildOne(ctx, source, e.Source.Path, followedBranch(e.Source.Ref), 0); err != nil { - state.State = "failed" - state.Why = err.Error() - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err) - continue - } - state.State = "asked" - state.AskedAt = &now + askModule(ctx, p, name, byName) } return nil } +// askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A +// variable so a test of what a plan does around an ask needs no build machine. +var askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) { + return buildOneAsked(ctx, source, path, ref, 0, false) +} + +// askModule asks the build machine for one module of a plan and marks it asked, with the id it was +// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked. +func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) { + now := time.Now().UTC() + state := p.Modules[name] + if state == nil { + state = &inventory.PlanModule{} + p.Modules[name] = state + } + e, known := byName[name] + if !known { + state.State = "failed" + state.Why = "no longer in the catalogue" + p.State = inventory.PlanFailed + p.Note = name + " is no longer in the catalogue" + return + } + source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat} + fmt.Printf(" tier %d: ", p.Tier) + // The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215). + id, err := askABuild(ctx, source, e.Source.Path, followedBranch(e.Source.Ref)) + if err != nil { + state.State = "failed" + state.Why = err.Error() + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err) + return + } + state.State = "asked" + state.AskedAt = &now + state.Build = id + state.Why, state.Commit, state.BuiltAt = "", "", nil +} + // planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and // advances what that completes. Called from the daemon's take-in of every outcome. // @@ -363,7 +378,7 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e // the later plan's answer — it stood on the bases from before the later plan's merge, and taking it // would send machines, and the next tier, what the later merge replaced. asked is zero when the // build's request time is not known, and such an outcome is taken as before. -func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time) { +func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) { inv := open.inventory // One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather // than write over what the holder is about to save. Not taken, it is still in the build records, @@ -399,7 +414,17 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string, state = &inventory.PlanModule{} p.Modules[module] = state } - if askedBefore(asked, state.AskedAt) { + // **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module + // is, as before, when it was asked at or after the plan's ask (issue 219). + // **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR + // 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's + // build, made from other source, and settling the plan with it would send that. A plan from + // before ids were kept is matched as it was: by when the build was asked (issue 219). + if state.Build != "" { + if state.Build != id { + continue + } + } else if askedBefore(asked, state.AskedAt) { continue } if failed != "" { @@ -523,6 +548,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, // plan never hears it. The record is the fact; a build recorded after the ask is that tier's // outcome, whoever was listening. recorded := map[string][]inventory.Build{} + byID := map[string]inventory.Build{} for _, m := range tier { if s := p.Modules[m]; s != nil && s.State == "asked" { builds, err := inv.Builds(ctx, m, 5) @@ -530,9 +556,19 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return false, err } recorded[m] = builds + // Its own ask's record, by id — found even when the outcome named no module (ADR 0219). + if s.Build != "" { + b, found, err := inv.BuildByID(ctx, s.Build) + if err != nil { + return false, err + } + if found { + byID[s.Build] = b + } + } } } - if settleFromRecords(p, tier, recorded) { + if settleFromRecords(p, tier, recorded, byID) { return true, nil } // Asked: wait for every build. @@ -811,7 +847,11 @@ func planTicker(ctx context.Context, open *stores) { } // planLine is one plan as `status` says it. -func planLine(p inventory.Plan, now time.Time) string { +func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}) } + +// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan +// waiting on builds nobody will take until a person resumes the seat says so, and is not late. +func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string { where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers)) switch p.State { case inventory.PlanDone: @@ -822,6 +862,9 @@ func planLine(p inventory.Plan, now time.Time) string { return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note) } since := now.Sub(p.Updated).Round(time.Second) + if waiting, paused := pausedWaiting(p, pause, now); paused { + return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting) + } late := "" if since > planWaitBound { late = " — LATE" @@ -835,20 +878,49 @@ func planLine(p inventory.Plan, now time.Time) string { // planFailedBuild marks the module a failed build was for when the result names no module: by the // repository and path the plan's modules were asked at. +// +// **By the id first** (novox/hq ADR 0219): a plan keeps the id it asked each module under, so an +// outcome that never learnt its module's name — cancelled, killed, failed at the clone — is matched +// to the module it was asked for exactly. Repository and path remain for a plan from before ids +// were kept. func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) { + asked, _ := link.BuildAskedAt(result.ID) + if plans, err := open.inventory.OpenPlans(ctx); err == nil { + if module := moduleAskedAs(plans, result.ID); module != "" { + planBuilt(ctx, open, module, result.Commit, result.Failed, asked, result.ID) + return + } + } entries, err := open.inventory.Catalogued(ctx) if err != nil { return } for _, e := range entries { if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path { - asked, _ := link.BuildAskedAt(result.ID) - planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked) + planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked, result.ID) return } } } +// moduleAskedAs is the module an open plan's current tier asked for under this id, or nothing. +func moduleAskedAs(plans []inventory.Plan, id string) string { + if id == "" { + return "" + } + for _, p := range plans { + if p.Tier >= len(p.Tiers) { + continue + } + for _, m := range p.Tiers[p.Tier] { + if s := p.Modules[m]; s != nil && s.Build == id { + return m + } + } + } + return "" +} + func repositoryMatches(a, b string) bool { trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) } return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a)) @@ -867,7 +939,7 @@ type planStatus struct { Late bool `json:"late"` } -func planStatuses(plans []inventory.Plan, now time.Time) []planStatus { +func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []planStatus { out := make([]planStatus, 0, len(plans)) for _, p := range plans { ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State, @@ -878,6 +950,10 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus { ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier) } ps.Late = now.Sub(p.Updated) > planWaitBound + // Paused is a person's decision, not lateness (novox/hq ADR 0219). + if waiting, paused := pausedWaiting(p, pause, now); paused { + ps.Waiting, ps.Late = waiting, false + } } out = append(out, ps) } @@ -885,12 +961,15 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus { } // openPlans is the open plans among the recent ones, and how many have waited past the bound. -func openPlans(plans []inventory.Plan) ([]inventory.Plan, int) { +func openPlans(plans []inventory.Plan, pause pauseView) ([]inventory.Plan, int) { var open []inventory.Plan late := 0 for _, p := range plans { if p.Open() { open = append(open, p) + if _, paused := pausedWaiting(p, pause, time.Now()); paused { + continue + } if time.Since(p.Updated) > planWaitBound { late++ } @@ -923,7 +1002,7 @@ func plansCommand(ctx context.Context, args []string) error { if err != nil { return err } - fmt.Printf("%s — %s\n", p.ID, planLine(p, now)) + fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}))) for i, tier := range p.Tiers { marker := " " if i == p.Tier && p.Open() { @@ -938,6 +1017,9 @@ func plansCommand(ctx context.Context, args []string) error { if s.Commit != "" { state += " from " + short(s.Commit) } + if s.Build != "" && s.State != "built" { + state += " (" + s.Build + ")" + } if s.Why != "" { state += ": " + s.Why } @@ -950,6 +1032,15 @@ func plansCommand(ctx context.Context, args []string) error { if *whatIf != "" { return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules)) } + // `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on. + if len(positionals) == 2 && positionals[0] == "retry" { + said, err := retryPlan(ctx, open, positionals[1]) + if err != nil { + return err + } + fmt.Println(said) + return nil + } // `stop`, or `close` (novox/hq issue 254): a person ending a plan that will not move again — one // waiting on a report that cannot come — so it stops reading as work in progress. Marked failed // with who ended it; what it asked still builds and registers. @@ -991,8 +1082,9 @@ func plansCommand(ctx context.Context, args []string) error { fmt.Println("no merge has produced a plan yet") return nil } + pause := buildSeatPause(ctx, inv, plans) for _, p := range plans { - fmt.Printf("%-28s %s\n", p.ID, planLine(p, now)) + fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause)) } return nil } @@ -1094,7 +1186,12 @@ func splitList(s string) []string { // settleFromRecords marks every module of the tier still `asked` built — or failed — from a build // recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214). // Newest first, as Builds answers: the first record after the ask is the outcome of that ask. -func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build) bool { +// +// The record of the plan's own ask, by its id, is that outcome before anything else (novox/hq ADR +// 0219): the plan asked under it, and a failure recorded without a module — cancelled, killed — is +// found by nothing else. +func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build, + byID map[string]inventory.Build) bool { changed := false for _, m := range tier { s := p.Modules[m] @@ -1103,6 +1200,10 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i } var outcome *inventory.Build for i := range recorded[m] { + // Asked under an id, only that id's record answers (ADR 0219), and it is looked up below. + if s.Build != "" { + break + } b := recorded[m][i] if b.At.Before(*s.AskedAt) { break @@ -1114,6 +1215,9 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i } outcome = &b } + if own, found := byID[s.Build]; s.Build != "" && found { + outcome = &own + } if outcome == nil { continue } diff --git a/cmd/mesh-controller/release_plan_test.go b/cmd/mesh-controller/release_plan_test.go index 365cdfc..e42c214 100644 --- a/cmd/mesh-controller/release_plan_test.go +++ b/cmd/mesh-controller/release_plan_test.go @@ -145,7 +145,7 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) { // Only a build from before the ask: not this ask's outcome. "builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}}, } - if !settleFromRecords(&p, p.Tiers[0], records) { + if !settleFromRecords(&p, p.Tiers[0], records, nil) { t.Fatal("nothing settled, though the controller's build is recorded after the ask") } if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil { @@ -162,13 +162,13 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) { late := map[string][]inventory.Build{"postgres": { {ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)}, }} - if settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "asked" { + if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" { t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"]) } // Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it. late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73", Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)}) - if !settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "built" || + if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" || r.Modules["postgres"].Commit != "4bcd5f73" { t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"]) } @@ -176,7 +176,7 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) { // A failure recorded after the ask fails the plan, as hearing it would have. q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}}, Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}} - settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}) + settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil) if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" { t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"]) } diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index d7dec42..a3fdfc0 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -103,10 +103,48 @@ func argvFor(verb string, args map[string]any) ([]string, error) { if id := str("close"); id != "" { return []string{"plans", "close", id}, nil } + if id := str("retry"); id != "" { + return []string{"plans", "retry", id}, nil + } if id := str("id"); id != "" { return []string{"plans", id}, nil } return []string{"plans"}, nil + // The build queue (novox/hq ADR 0219). + case "queue": + return []string{"queue"}, nil + case "cancel", "kill": + if err := need("id"); err != nil { + return nil, err + } + return []string{verb, str("id")}, nil + case "clear": + if str("dead") == "true" { + return []string{"clear", "--dead"}, nil + } + return []string{"clear"}, nil + case "rebuild": + if err := need("what"); err != nil { + return nil, err + } + return []string{"rebuild", str("what")}, nil + case "replay": + if err := need("id"); err != nil { + return nil, err + } + argv := []string{"replay", str("id")} + if str("register") == "true" { + argv = append(argv, "--register") + } + if str("older") == "true" { + argv = append(argv, "--older") + } + return argv, nil + case "pause", "resume": + if n := str("node"); n != "" { + return []string{verb, n}, nil + } + return []string{verb}, nil case "plan": if err := need("node"); err != nil { return nil, err diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index 9756347..7cf2141 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -138,14 +138,14 @@ func printStatus(asked answers) error { len(quiet), strings.Join(said, "\n ")) } - if open, late := openPlans(asked.plans); len(open) > 0 { + if open, late := openPlans(asked.plans, asked.paused); len(open) > 0 { fmt.Printf("%d plan(s) open", len(open)) if late > 0 { fmt.Printf(", %d waiting past %s", late, planWaitBound) } fmt.Println(":") for _, p := range open { - fmt.Printf(" %s\n", planLine(p, time.Now())) + fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.paused)) } fmt.Println() } @@ -420,6 +420,8 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { if err != nil { return answers{}, err } + // Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219). + out.paused = buildSeatPause(ctx, inv, out.plans) // And which machines are not running what the mesh would send them. The same question as a // module being behind its source, one level down: that one says the catalogue is out of date, diff --git a/internal/broker/cancelled.go b/internal/broker/cancelled.go new file mode 100644 index 0000000..afd0008 --- /dev/null +++ b/internal/broker/cancelled.go @@ -0,0 +1,92 @@ +package broker + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// A seat's cancelled set (novox/hq ADR 0219). +// +// **Cancelling an ask is deleting its message from the seat's work queue** — and that alone has a +// race no ordering on one side closes: a holder may fetch the ask in the moment between the +// controller reading the queue and deleting the message, and build what a person cancelled. So the +// controller first writes the ask's id into the seat's cancelled set, and a holder looks its ask up +// there after taking it and before building: listed, it terminates the ask and announces it failed +// rather than starting it. +// +// **A key-value bucket, because that is the shape the bus already has for "a small set the +// controller writes and many read cheaply"** (ADR 0201's state buckets): one direct read by key per +// ask taken — no consumer, no subscription a holder must keep, nothing that grows a holder's grants +// beyond one read subject. Kept beside the seat's own stream and worker and named like them, so the +// three objects of one work queue read as one family; asserted by the controller with the queue, +// and aged out with it — an id cancelled a week ago names an ask the queue no longer holds. + +// CancelledSetName is the bucket holding a seat's cancelled asks. +func CancelledSetName(seat string) string { return "SEAT_" + upperSnake(seat) + "_cancelled" } + +// hasCancelledSet is whether a seat's queue can be cancelled from: the work queues the controller +// asks, whose asks it alone shows and changes. A module's own seat's queue is that module's affair. +func hasCancelledSet(seat string) bool { + for _, s := range seatsTheControllerAsks { + if s == seat { + return true + } + } + return false +} + +// IsCancelledSet says a bucket is a seat's cancelled set rather than a module's state — the mesh's +// own, and never one to report as state nothing declares. +func IsCancelledSet(bucket string) bool { + return strings.HasPrefix(bucket, "SEAT_") && strings.HasSuffix(bucket, "_cancelled") +} + +// cancelledSetAge is how long a cancelled id is kept: as long as the seat's queue keeps an ask. +const cancelledSetAge = 7 * 24 * time.Hour + +// A CancelledSetAsserter is what raising the cancelled sets needs of a connection. +type CancelledSetAsserter interface { + EnsureCancelledSet(seat string) error +} + +// RaiseCancelledSets asserts the cancelled set of every work queue the controller asks. +func RaiseCancelledSets(a CancelledSetAsserter, seats []DeclaredSeat) error { + for _, s := range seats { + if len(s.Accepts) == 0 || !hasCancelledSet(s.Name) { + continue + } + if err := a.EnsureCancelledSet(s.Name); err != nil { + return fmt.Errorf("asserting the cancelled set of %s: %w", s.Name, err) + } + } + return nil +} + +// EnsureCancelledSet creates a seat's cancelled set if absent and brings its options to match. +// Direct reads on, which is how a holder looks an id up with one request. +func (j *JetStream) EnsureCancelledSet(seat string) error { + js, err := jetstream.New(j.conn) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: CancelledSetName(seat), + Description: fmt.Sprintf("the asks of the %s seat cancelled by hand (novox/hq ADR 0219): written "+ + "by the controller before it deletes an ask from the queue, read by a holder on taking one, "+ + "so an ask fetched in that moment is ended rather than built", seat), + History: 1, + TTL: cancelledSetAge, + MaxValueSize: 4 * 1024, + MaxBytes: 4 * 1024 * 1024, + Storage: jetstream.FileStorage, + }); err != nil { + return fmt.Errorf("asserting bucket %s: %w", CancelledSetName(seat), err) + } + return nil +} diff --git a/internal/broker/cancelled_test.go b/internal/broker/cancelled_test.go new file mode 100644 index 0000000..5bf4a0a --- /dev/null +++ b/internal/broker/cancelled_test.go @@ -0,0 +1,73 @@ +package broker + +import "testing" + +// A build agent reads its seat's cancelled set by key and nothing more, answers its verbs on its own +// machine's subjects, and says whether it is paused under its own machine's name; the controller may +// ask any machine's holder its verbs (novox/hq ADR 0219). +func TestTheBuildQueueIsControlledWithTheGrantsItNeedsAndNoMore(t *testing.T) { + seat := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, + Emits: []string{"started", "built", "log.*", "paused.*"}, + Serves: []string{"current", "kill", "pause", "resume"}} + holder, err := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "build-agent", + Holds: []Seat{seat}, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + has(t, holder.Publish, "$JS.API.DIRECT.GET.KV_SEAT_NODE_BUILD_AGENT_cancelled.$KV.SEAT_NODE_BUILD_AGENT_cancelled.>") + hasNot(t, holder.Publish, "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>") + has(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.ace") + hasNot(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.*") + has(t, holder.Publish, "mesh.seat.node-build-agent.event.log.*") + has(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.ace") + hasNot(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.g14") + + // A module's own seat's queue is its own affair: no cancelled set, no grant for one. + other, _ := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "telegram", + Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, PasswordHash: "x"}) + hasNot(t, other.Publish, "$JS.API.DIRECT.GET.KV_SEAT_TELEGRAM_SENDER_cancelled.$KV.SEAT_TELEGRAM_SENDER_cancelled.>") + + controller, _ := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) + has(t, controller.Publish, "mesh.seat.node-build-agent.tool.>") + + if CancelledSetName("node-build-agent") != "SEAT_NODE_BUILD_AGENT_cancelled" || + !IsCancelledSet("SEAT_NODE_BUILD_AGENT_cancelled") || IsCancelledSet("build-agent_cancelled") { + t.Error("the cancelled set is not named as its seat's family") + } +} + +// Only the work queues the controller asks get a cancelled set. +func TestOnlyTheControllersQueuesHaveACancelledSet(t *testing.T) { + var asserted []string + a := asserterFunc(func(seat string) error { asserted = append(asserted, seat); return nil }) + if err := RaiseCancelledSets(a, []DeclaredSeat{ + {Name: "node-build-agent", Accepts: []string{"build"}}, + {Name: "telegram-sender", Accepts: []string{"send"}}, + {Name: "mesh-controller"}, + }); err != nil { + t.Fatal(err) + } + if len(asserted) != 1 || asserted[0] != "node-build-agent" { + t.Fatalf("asserted %v", asserted) + } +} + +type asserterFunc func(string) error + +func (f asserterFunc) EnsureCancelledSet(seat string) error { return f(seat) } + +// A seat's cancelled set is never reported as state nothing declares. +func TestACancelledSetIsNotUndeclaredState(t *testing.T) { + undeclared, err := RaiseBuckets(fakeBuckets{names: []string{"SEAT_NODE_BUILD_AGENT_cancelled", "gone_state"}}, nil) + if err != nil { + t.Fatal(err) + } + if len(undeclared) != 1 || undeclared[0] != "gone_state" { + t.Fatalf("undeclared %v", undeclared) + } +} + +type fakeBuckets struct{ names []string } + +func (f fakeBuckets) EnsureBucket(Bucket) error { return nil } +func (f fakeBuckets) BucketNames() ([]string, error) { return f.names, nil } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 556f8e3..2cca06d 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -124,6 +124,10 @@ type Principal struct { // goes with the retired seat row. var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"} +// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the +// holder's machine (novox/hq ADR 0219): `paused.`, the build agent saying whether it takes work. +var perMachineEvents = map[string]bool{"paused.*": true} + // enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the // controller may answer an enrolment is derived from the same constant the user is named from. const enrolmentPrefix = "enrol" @@ -221,6 +225,10 @@ func PermissionsFor(p Principal) (Permissions, error) { // the role, and whichever machine holding it is idle takes it. for _, seat := range seatsTheControllerAsks { pub = append(pub, "mesh.seat."+seat+".accept.>") + // **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is + // building, kill it, pause it, resume it. The queue is the controller's to show and to + // change, and what one machine is doing with an ask it took only that machine can say. + pub = append(pub, "mesh.seat."+seat+".tool.>") } // **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own // facts under the seat it holds, because a role's events belong to the role and keep their @@ -404,10 +412,26 @@ func PermissionsFor(p Principal) (Permissions, error) { "$JS.API.CONSUMER.INFO."+stream+"."+worker, "$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker, "$JS.ACK."+stream+"."+worker+".>") + // **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the + // seat's cancelled set, read directly by its id, so a holder that fetched an ask in the + // moment the controller cancelled it ends it instead of building it. Read, never written: + // the set is the controller's, and only the work queues the controller asks — and so may + // cancel from — have one. + if hasCancelledSet(s.Name) { + set := CancelledSetName(s.Name) + pub = append(pub, "$JS.API.DIRECT.GET.KV_"+set+".$KV."+set+".>") + } for _, a := range s.Accepts { sub = append(sub, seatSubject(s, "accept", a)) } for _, e := range s.Emits { + // **A machine says its own state and no other's** (novox/hq ADR 0219): on a node-scoped + // seat, an event about the holder itself carries the machine as its last token, and + // each holder is granted its own machine's alone. + if s.Scope == "node" && p.Node != "" && perMachineEvents[e] { + pub = append(pub, seatSubject(s, "event", strings.TrimSuffix(e, "*")+p.Node)) + continue + } pub = append(pub, seatSubject(s, "event", e)) } for _, t := range s.Serves { diff --git a/internal/broker/state.go b/internal/broker/state.go index 0324543..9c1e594 100644 --- a/internal/broker/state.go +++ b/internal/broker/state.go @@ -160,7 +160,8 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err return nil, fmt.Errorf("listing the bus's state: %w", err) } for _, n := range names { - if !declared[n] { + // A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state. + if !declared[n] && !IsCancelledSet(n) { undeclared = append(undeclared, n) } } diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 4c7b32e..af658e6 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] } subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 5f26afa..1441265 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -761,6 +761,9 @@ func Command(ctx context.Context, dir, name string, args ...string) (string, err tell("run", "$ (%s) %s %s", short(filepath.Base(dir)), name, strings.Join(args, " ")) cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = dir + // **Ended whole when the build is** (novox/hq ADR 0219): its own process group, killed as one + // when the context ends, so a build killed by hand leaves no `git` or `docker` child running. + inItsOwnGroup(cmd) out, err := cmd.CombinedOutput() if err != nil { tell("run", "! %s %s failed after %s", name, args[0], since(started)) diff --git a/internal/builder/kill.go b/internal/builder/kill.go new file mode 100644 index 0000000..a46dec2 --- /dev/null +++ b/internal/builder/kill.go @@ -0,0 +1,78 @@ +package builder + +import ( + "context" + "os/exec" + "strings" + "syscall" + "time" +) + +// A build killed by hand (novox/hq ADR 0219). +// +// A build is a tree of commands — git, then docker, and docker's own children — and a container the +// docker client started keeps running when the client dies. So ending one by hand is three things: +// the build's context is cancelled, which kills each command's whole process group rather than the +// one process the context knows; every container the build started carries the build's id as a +// label, so what outlived its client is found and removed by that label; and the holder announces +// the outcome itself, since nothing else will. + +// BuildLabel is the label every container a build starts carries, valued with the build's id. +const BuildLabel = "mesh.build" + +// KillWait is how long a command killed with its build may take to let go of its output before it +// is abandoned: a grandchild holding the pipe open must not hold the build open with it. +const KillWait = 10 * time.Second + +// inItsOwnGroup makes a command the leader of its own process group, killed as a group when its +// context ends. +func inItsOwnGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if cmd.Process == nil { + return nil + } + // The negative pid is the group: the command and everything it started. + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { + return cmd.Process.Kill() + } + return nil + } + cmd.WaitDelay = KillWait +} + +// Labelled is a Runner that marks every container a build starts with the build's id, so a kill +// finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command +// passes rather than at each `docker run` the builder composes: a run added later is labelled too. +func Labelled(run Runner, id string) Runner { + return func(ctx context.Context, dir string, name string, args ...string) (string, error) { + return run(ctx, dir, name, LabelledArgs(name, args, id)...) + } +} + +// LabelledArgs is a command's arguments with the build's label added, when it is a `docker run`. +func LabelledArgs(name string, args []string, id string) []string { + if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" { + return args + } + out := make([]string, 0, len(args)+2) + out = append(out, "run", "--label", BuildLabel+"="+id) + return append(out, args[1:]...) +} + +// RemoveContainersOf removes every container labelled with the build's id, running or not, and +// says how many. Run with a context of its own: the build's is the one that was just cancelled. +func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error) { + out, err := run(ctx, "", "docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id) + if err != nil { + return 0, err + } + ids := strings.Fields(out) + if len(ids) == 0 { + return 0, nil + } + if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil { + return 0, err + } + return len(ids), nil +} diff --git a/internal/builder/kill_test.go b/internal/builder/kill_test.go new file mode 100644 index 0000000..03a7c90 --- /dev/null +++ b/internal/builder/kill_test.go @@ -0,0 +1,103 @@ +package builder + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// A build killed by hand (novox/hq ADR 0219) ends every command it started: the context's end kills +// the command's whole process group, not the one process the context knows about. +func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) { + dir := t.TempDir() + child := filepath.Join(dir, "child") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + began := time.Now() + go func() { + // A shell that starts a grandchild and waits on it: killing the shell alone would leave the + // grandchild running, holding the output open. + _, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`) + done <- err + }() + var pid int + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { + if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" { + pid, _ = strconv.Atoi(strings.TrimSpace(string(raw))) + break + } + } + if pid == 0 { + t.Fatal("the command never started its child") + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("a killed command reported success") + } + case <-time.After(KillWait + 5*time.Second): + t.Fatal("the killed command never returned") + } + if took := time.Since(began); took > KillWait { + t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took) + } + for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return + } + } + _ = syscall.Kill(pid, syscall.SIGKILL) + t.Fatalf("the grandchild %d outlived the kill", pid) +} + +// Every `docker run` a build starts carries its id as a label; nothing else is touched. +func TestEveryContainerABuildStartsCarriesItsID(t *testing.T) { + got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1") + if want := []string{"run", "--label", "mesh.build=build-1", "--rm", "img", "sh"}; !reflect.DeepEqual(got, want) { + t.Errorf("docker run became %v", got) + } + for _, c := range []struct { + name string + args []string + }{{"docker", []string{"build", "."}}, {"git", []string{"run"}}, {"docker", nil}} { + if got := LabelledArgs(c.name, c.args, "build-1"); !reflect.DeepEqual(got, c.args) { + t.Errorf("%s %v became %v", c.name, c.args, got) + } + } + var ran [][]string + run := Labelled(func(_ context.Context, _ string, name string, args ...string) (string, error) { + ran = append(ran, append([]string{name}, args...)) + return "", nil + }, "build-2") + _, _ = run(context.Background(), "", "docker", "run", "img") + if want := [][]string{{"docker", "run", "--label", "mesh.build=build-2", "img"}}; !reflect.DeepEqual(ran, want) { + t.Errorf("ran %v", ran) + } +} + +// What a kill removes is found by the label, and only what it finds. +func TestAKillRemovesTheContainersLabelledWithTheBuild(t *testing.T) { + var ran []string + run := func(_ context.Context, _ string, name string, args ...string) (string, error) { + ran = append(ran, name+" "+strings.Join(args, " ")) + if args[0] == "ps" { + return "c1\nc2\n", nil + } + return "", nil + } + n, err := RemoveContainersOf(context.Background(), run, "build-3") + if err != nil || n != 2 { + t.Fatalf("%d %v", n, err) + } + if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker rm -f c1 c2"}; !reflect.DeepEqual(ran, want) { + t.Errorf("ran %v", ran) + } +} diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 39a35ef..7764096 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -117,8 +117,16 @@ var defaultSeats = append([]Seat{ // **Node-scoped, and every holder takes from one queue** (novox/hq ADR 0190): a build is asked of // the role, and whichever machine holding the seat is idle pulls it. One holder per machine is // what the scope says; sharing the work is what a seat's queue has always done. + // + // **And its holder answers for the build it is running** (novox/hq ADR 0219): what it is + // building, kill it, take nothing new, take again. The queue as a whole is the controller's to + // show and change (`queue`, `cancel`, `clear`); what one machine does with an ask it already + // took only that machine can do. `paused.` is each holder saying whether it takes work, so + // the controller can tell a plan waiting on a paused seat from one that is late. {Name: "node-build-agent", Scope: ScopeNode, - Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"}, + Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*", "paused.*"}, + Serves: buildAgentVerbs(), + Decision: "novox/hq ADR 0190, ADR 0219"}, // **Retired by ADR 0190, kept while a manifest still claims it.** The one build machine's seat. // A claim to a seat the mesh no longer defines is refused, and the module holding this one is // assigned on a live machine until build-agent replaces it — removing the row first would make @@ -581,3 +589,23 @@ func loginShellVerbs() []Verb { }, []string{"command"})}, } } + +// buildAgentVerbs are what a holder of node-build-agent answers on its own machine (novox/hq ADR +// 0219). One build at a time per holder (ADR 0190), so "the build running here" is one or none. +func buildAgentVerbs() []Verb { + return []Verb{ + {Name: "current", Description: "The build this machine is running — its id, repository, path, " + + "the step it is at, when it started and for how long — or none; and whether this machine is " + + "paused, taking no new build.", + Input: schema(map[string]string{}, nil)}, + {Name: "kill", Description: "End the build with this id, running here: its commands and the " + + "containers it started are stopped, and its outcome is announced as failed, killed by hand — " + + "settled, so it is not handed to another machine.", + Input: schema(map[string]string{"id": "the build's id, as `queue` or `current` says it"}, []string{"id"})}, + {Name: "pause", Description: "Take no new build on this machine until resumed; a build running " + + "here finishes. Kept across a restart of the holder.", + Input: schema(map[string]string{}, nil)}, + {Name: "resume", Description: "Take builds again on this machine.", + Input: schema(map[string]string{}, nil)}, + } +} diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 8898585..decb038 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -97,6 +97,7 @@ var ControllerVerbs = []Verb{ "id": "a plan's id (as `plans` lists them): that plan, tier by tier", "stop": "a plan's id: stop it — what was asked still builds, nothing further is asked", "close": "a plan's id: close a plan that will not move again, as failed by hand (novox/hq issue 254)", + "retry": "a failed plan's id: ask its failed builds again under new ids, and carry the plan on from that tier (novox/hq ADR 0219)", "repository": "owner/repository: the plan a merge there would produce, saving nothing (what-if); with paths or modules", "paths": "with repository: the files the merge would change, comma-separated, from the repository's root", "modules": "with repository: or the modules it would change, comma-separated", @@ -156,6 +157,36 @@ var ControllerVerbs = []Verb{ Input: schema(map[string]string{ "command": "the command line, as the controller's binary takes it; quotes group a word with spaces", }, []string{"command"})}, + // The build queue, controlled by hand (novox/hq ADR 0219). Every verb that drops an ask leaves a + // failed outcome for it, so a plan waiting on it fails visibly instead of hanging. + {Name: "queue", Description: "Every ask in the build queue: waiting, in flight (on which machine, for how long), " + + "and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked.", + Input: schema(nil, nil)}, + {Name: "cancel", Description: "Drop one waiting or dead ask from the build queue; its outcome is recorded failed, " + + "cancelled by hand, and a plan that asked for it fails. One in flight is refused: `kill` ends it where it runs.", + Input: schema(map[string]string{"id": "the ask's build id, as `queue` lists it"}, []string{"id"})}, + {Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " + + "recorded failed, cancelled by hand. Never touches one in flight.", + Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil)}, + {Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " + + "build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.", + Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})}, + {Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id. " + + "A dry run unless register: nothing recorded or registered. Registering is refused when a newer build of the module " + + "is registered — it would roll the older commit out (novox/hq issue 207) — unless older says so.", + Input: schema(map[string]string{ + "id": "the build's id", + "register": "\"true\" to register what it builds", + "older": "\"true\", with register: even though a newer build of the module is registered", + }, []string{"id"})}, + {Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " + + "announces it failed, killed by hand — settled, never handed to another machine.", + Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})}, + {Name: "pause", Description: "The build seat's holder on one machine — or every holder — takes no new build until resumed; " + + "a build running finishes. Kept across a restart of the holder. A plan waiting on a paused seat says so and is not late.", + Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)}, + {Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.", + Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)}, {Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " + "`builds` with that id follows it line by line, and the module is registered when the outcome comes.", Input: schema(map[string]string{ diff --git a/internal/inventory/builds.go b/internal/inventory/builds.go index 5400acf..082fce2 100644 --- a/internal/inventory/builds.go +++ b/internal/inventory/builds.go @@ -3,8 +3,11 @@ package inventory import ( "context" "encoding/json" + "errors" "sort" "time" + + "github.com/jackc/pgx/v5" ) // What has been built. @@ -161,6 +164,33 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui return out, rows.Err() } +// BuildByID is one build's record, by the id its request carried — what a plan matches its outcome +// by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the +// repository, path, ref and commit from. False when no outcome with that id was recorded. +func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) { + var b Build + var made []byte + var asked *time.Time + err := i.store.Pool().QueryRow(ctx, + `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, + coalesce(source_path,''), asked, at + from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, + &b.On, &b.Failed, &made, &b.Path, &asked, &b.At) + if errors.Is(err, pgx.ErrNoRows) { + return Build{}, false, nil + } + if err != nil { + return Build{}, false, err + } + if asked != nil { + b.Asked = *asked + } + if err := json.Unmarshal(made, &b.Made); err != nil { + return Build{}, false, err + } + return b, true, nil +} + // Held is every artifact this mesh has built, keyed "/". // // **The successful build of each module asked last wins**, which is the same rule the rest of the diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index cf90380..50e8b89 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -49,6 +49,11 @@ type PlanModule struct { FirstAt *time.Time `json:"first_at,omitempty"` Commit string `json:"commit,omitempty"` Why string `json:"why,omitempty"` + // Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan + // matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt + // its module's name included. Empty in a plan from before it was kept, which is matched by + // module, or by repository and path, as before. + Build string `json:"build,omitempty"` } // The states a plan passes through. diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go index 21bdca4..f5d1c4c 100644 --- a/internal/link/builds_nats.go +++ b/internal/link/builds_nats.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "os" "time" "github.com/nats-io/nats.go" @@ -105,7 +106,20 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest, waiting, cancelWait := context.WithTimeout(ctx, wait) defer cancelWait() for { - msg, err := outcomes.NextMsgWithContext(waiting) + // **A wait that hears a cancel** (novox/hq ADR 0219). A person cancelling an ask records its + // failure without publishing the role's outcome — that subject is the holders', and the + // controller may not speak for them — so the waiter also looks, every while, at the cancelled + // set the cancel wrote first. A kill needs no look: the holder announces it as any outcome. + slice, endSlice := context.WithTimeout(waiting, cancelLook) + msg, err := outcomes.NextMsgWithContext(slice) + endSlice() + if errors.Is(err, context.DeadlineExceeded) && waiting.Err() == nil { + if cancelled, _ := IsCancelled(b.js.Conn(), b.role(), request.ID); cancelled { + return BuildResult{ID: request.ID, Repository: request.Repository, Path: request.Path, + Ref: request.Ref, Source: request.Source, DryRun: request.DryRun, Failed: CancelledByHand}, nil + } + continue + } switch { case errors.Is(err, context.DeadlineExceeded): return BuildResult{}, waitingFor(wait) @@ -124,6 +138,9 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest, } } +// cancelLook is how often a waiting asker looks whether its ask was cancelled. +var cancelLook = 2 * time.Second + // --- the machine's side --------------------------------------------------------------------- type natsMachine struct { @@ -131,6 +148,14 @@ type natsMachine struct { on string seat string sub *nats.Subscription + opts MachineOptions +} + +// MachineOptions is what a holder tells the taking loop about itself (novox/hq ADR 0219). +type MachineOptions struct { + // Paused is asked before every fetch: while it says so, nothing new is taken, and a build + // already running finishes. Nil is never paused. + Paused func() bool } // MachineOverNATS takes build work from the current build role. @@ -145,6 +170,18 @@ func MachineOverNATSOn(js *broker.JetStream, on, seat string) BuildMachine { return &natsMachine{js: js, on: on, seat: seat} } +// MachineOverNATSWith is MachineOverNATSOn for a holder that can be paused (novox/hq ADR 0219). +func MachineOverNATSWith(js *broker.JetStream, on, seat string, opts MachineOptions) BuildMachine { + return &natsMachine{js: js, on: on, seat: seat, opts: opts} +} + +// pausedPoll is how often a paused holder looks again whether it was resumed, and pausedFetch how +// long one pull of a holder that can be paused waits. +const ( + pausedPoll = 2 * time.Second + pausedFetch = 5 * time.Second +) + func (m *natsMachine) Close() { if m.sub != nil { _ = m.sub.Unsubscribe() @@ -189,9 +226,27 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) if ctx.Err() != nil { return nil } + // **Paused takes nothing new** (novox/hq ADR 0219). Asked before the fetch, never during a + // build: what this machine already took it finishes, and what it has not taken stays in the + // queue for another holder — or for this one, resumed. + if m.opts.Paused != nil && m.opts.Paused() { + select { + case <-ctx.Done(): + return nil + case <-time.After(pausedPoll): + } + continue + } // One, and wait a while for it; an empty queue is a timeout, which is the normal state of a // machine with nothing to build, and is asked again. - fetched, err := sub.Fetch(1, nats.Context(ctx)) + // Asked for a few seconds at a time when the holder can be paused, so a pause reaches a pull + // already waiting within that, rather than when the client's own wait runs out. + asking, endAsking := ctx, func() {} + if m.opts.Paused != nil { + asking, endAsking = context.WithTimeout(ctx, pausedFetch) + } + fetched, err := sub.Fetch(1, nats.Context(asking)) + endAsking() switch { case ctx.Err() != nil: // Ours ended: the machine is being stopped. @@ -213,6 +268,13 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) return fmt.Errorf("the bus stopped delivering build work: %w", err) } for _, msg := range fetched { + // **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for + // the one that arrived in that moment. Handed back at once for another holder — counted as + // a delivery, which a pause landing exactly then costs and nothing else does. + if m.opts.Paused != nil && m.opts.Paused() { + _ = msg.Nak() + continue + } var request BuildRequest if err := json.Unmarshal(msg.Data, &request); err != nil { // Unreadable: terminated rather than retried, because the next attempt reads the same @@ -220,12 +282,25 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) _ = msg.Term() continue } + // **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the + // id into the seat's cancelled set before deleting the ask, so one taken in between is + // ended here, terminated rather than redelivered, and its outcome said as failed — never + // built. A set that cannot be read is said and the ask built: a cancel is a person's + // exception, and a holder that refused every build while its grant was missing would + // stop the mesh building for it. + build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat} + if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil { + fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err) + } else if cancelled { + endCancelled(ctx, build) + continue + } // A build outlives the acknowledgement window many times over; said while it runs, // as the controller says it for its own long handlers, so the server neither hands // the ask to a second machine nor counts the wait against its deliveries. working := make(chan struct{}) go stillWorking(msg, working) - do(ctx, &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat}) + do(ctx, build) close(working) } } @@ -307,3 +382,17 @@ func (b *natsBuild) Say(step, message string) { } func (b *natsBuild) Hold(after time.Duration) error { return b.msg.NakWithDelay(after) } + +// endCancelled settles an ask that was cancelled as it was taken: its outcome said as failed, as the +// controller already recorded it, so anybody still waiting on it hears the answer — then +// terminated, so the queue neither keeps nor redelivers it. +func endCancelled(ctx context.Context, b *natsBuild) { + r := b.request + fmt.Fprintf(os.Stderr, "%s was cancelled by hand as this machine took it; not built\n", r.ID) + result := BuildResult{ID: r.ID, Repository: r.Repository, Path: r.Path, Ref: r.Ref, On: b.on, + Source: r.Source, DryRun: r.DryRun, Failed: CancelledByHand} + if err := b.Announce(ctx, result); err != nil { + fmt.Fprintf(os.Stderr, "cannot say %s was cancelled: %v\n", r.ID, err) + } + _ = b.msg.Term() +} diff --git a/internal/link/builds_nats_test.go b/internal/link/builds_nats_test.go index b0feaa2..fcbe211 100644 --- a/internal/link/builds_nats_test.go +++ b/internal/link/builds_nats_test.go @@ -303,8 +303,9 @@ func TestNatsTwoMachinesShareTheWorkAndNeitherIsHandedMoreThanItCanTake(t *testi case <-time.After(2 * time.Second): } // One finishes, and only then is the third taken — by that machine, the one that is free. - close(release["anchor"]) - release["anchor"] = make(chan struct{}) + // Released by a send, never by replacing the channel: the map is read by both machines' builds + // while this runs, and writing it raced them. + release["anchor"] <- struct{}{} select { case got := <-took: if got.machine != "anchor" { @@ -318,7 +319,7 @@ func TestNatsTwoMachinesShareTheWorkAndNeitherIsHandedMoreThanItCanTake(t *testi // an explicit hand-back, because the real wait is a minute. Here: the laptop goes, anchor // finishes, and with nothing queued nothing more is taken by the machine that is left. machines["laptop"].Close() - close(release["anchor"]) + release["anchor"] <- struct{}{} select { case got := <-took: t.Fatalf("%s took %s; the queue should be empty", got.machine, got.id) diff --git a/internal/link/queue.go b/internal/link/queue.go new file mode 100644 index 0000000..254dc52 --- /dev/null +++ b/internal/link/queue.go @@ -0,0 +1,465 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The build queue, read and changed by hand (novox/hq ADR 0219). +// +// **A queue nobody can see is a queue nobody can trust.** Until this, the build seat's work queue +// was visible only as its consequences: a plan LATE with nothing to say why, a build that never +// started because five asks ahead of it were waiting on a laptop that was shut. ADR 0219 makes the +// queue the controller's to show and change, and the build running on one machine that machine's +// holder's to end. +// +// Three kinds of ask are in the seat's stream at any moment, told apart by the worker consumer +// every holder pulls from: +// +// - **waiting** — after the last one the worker handed out (stream seq > delivered.stream_seq); +// - **in flight** — handed out and not yet settled, which the holder that took it said with its +// `started` event, and which the worker counts among its acks pending; +// - **dead** — handed out as many times as the worker allows and never settled. A work queue +// drops what it settles, so one still in the stream behind the worker is either in flight or +// dead; the started events and the worker's pending count say which. +// +// Everything that drops an ask leaves a failed outcome for it, recorded the way a failed build's is +// (`cancelled by hand`, `killed by hand`), so a plan waiting on it fails visibly rather than waiting +// for ever on an ask nobody will answer. + +// The words an outcome says when a person ended the ask. +const ( + CancelledByHand = "cancelled by hand" + KilledByHand = "killed by hand" +) + +// Ask states in a queue. +const ( + AskWaiting = "waiting" + AskInFlight = "in flight" + AskDead = "dead" +) + +// QueuedAsk is one ask in the seat's work queue. +type QueuedAsk struct { + Seq uint64 `json:"seq"` + Request BuildRequest `json:"-"` + ID string `json:"id"` + // Repository, Path and Ref are the ask's, as the request carried them; Held never travels out of + // here — it is every artifact the mesh has built, and a queue listing is not where that belongs. + Repository string `json:"repository"` + Path string `json:"path,omitempty"` + Ref string `json:"ref,omitempty"` + AskedAt time.Time `json:"asked-at,omitempty"` + State string `json:"state"` + // On and Started are the holder building an in-flight ask and when it started, from its started + // event. Was is the holder that started an in-flight ask and is no longer building it — handed + // back, to be handed out again — with Started its start there. + On string `json:"on,omitempty"` + Was string `json:"was-on,omitempty"` + Started time.Time `json:"started,omitempty"` +} + +// Queue is the seat's work queue as it stands. +type Queue struct { + Seat string `json:"seat"` + // Delivered is the last stream sequence the worker handed out; AckPending how many it handed out + // and has not had settled; MaxDeliver how many times it hands one ask out. + Delivered uint64 `json:"delivered"` + AckPending int `json:"ack-pending"` + MaxDeliver int `json:"max-deliver"` + Asks []QueuedAsk `json:"asks"` +} + +// Of is the asks in one state, in queue order. +func (q Queue) Of(state string) []QueuedAsk { + var out []QueuedAsk + for _, a := range q.Asks { + if a.State == state { + out = append(out, a) + } + } + return out +} + +// Find is the ask with this id. +func (q Queue) Find(id string) (QueuedAsk, bool) { + for _, a := range q.Asks { + if a.ID == id { + return a, true + } + } + return QueuedAsk{}, false +} + +// BuildHeard is what the seat's events say about builds: the latest start of each id, and every id +// whose outcome was heard. +type BuildHeard struct { + Started map[string]BuildStart + Outcomes map[string]bool +} + +// ClassifyQueue says which state each ask is in. Pure: the stream's asks in sequence order, what the +// worker says, and what the seat's events said. +// +// **In flight is what the worker counts handed out and unsettled**, at most AckPending of the asks +// behind it, given out in this order: +// +// 1. what a machine is building now — its latest start, no outcome heard (a holder builds one ask +// at a time, ADR 0190), newest start first; +// 2. what a machine started and is no longer building — a holder restarted mid-build, the ask +// handed back and waiting to be handed out again while it has deliveries left — newest start +// first, naming the machine it was last on; +// 3. what was taken and not yet said started. +// +// Only what is left after that is dead: so nothing behind the worker is dead while there are no more +// of them than the worker counts pending. A machine that died mid-build keeps a latest start for +// ever; the worker's count is what says the ask was handed out as often as it may be. +// +// **The worker's count is the server's, and it lags in one case**: an ask handed out its last time +// and handed back is still counted pending until some holder pulls again, when the server finds it +// past its deliveries and lets it go. Holders pull whenever they are idle, so this is a moment — +// except while every holder is paused, when such an ask reads as in flight with no machine named. +func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard BuildHeard) []QueuedAsk { + // Each machine's latest start. + latest := map[string]BuildStart{} + for _, s := range heard.Started { + at := startedAt(s) + if was, ok := latest[s.On]; !ok || at.After(startedAt(was)) { + latest[s.On] = s + } + } + current := map[string]BuildStart{} + for _, s := range latest { + if !heard.Outcomes[s.ID] { + current[s.ID] = s + } + } + + out := make([]QueuedAsk, len(asks)) + copy(out, asks) + var running, handedBack, unsaid []int + for i := range out { + a := &out[i] + if a.Seq > delivered { + a.State = AskWaiting + continue + } + a.State = AskDead + if s, ok := current[a.ID]; ok { + a.On, a.Started = s.On, startedAt(s) + running = append(running, i) + continue + } + if s, ever := heard.Started[a.ID]; ever { + a.Was, a.Started = s.On, startedAt(s) + handedBack = append(handedBack, i) + continue + } + unsaid = append(unsaid, i) + } + newestFirst := func(ix []int) { + sort.SliceStable(ix, func(x, y int) bool { return out[ix[x]].Started.After(out[ix[y]].Started) }) + } + newestFirst(running) + newestFirst(handedBack) + slots := ackPending + for _, group := range [][]int{running, handedBack, unsaid} { + for _, i := range group { + if slots == 0 { + // Past what the worker counts: handed out as often as it may be. + out[i].On, out[i].Started = "", time.Time{} + continue + } + out[i].State = AskInFlight + slots-- + } + } + return out +} + +func startedAt(s BuildStart) time.Time { + t, _ := time.Parse(time.RFC3339Nano, s.At) + return t +} + +// ReadQueue reads a build seat's work queue: the stream's asks, the worker's position, and what the +// seat's events said about the builds behind it. Through the JetStream API alone (STREAM.INFO, +// CONSUMER.INFO, STREAM.MSG.GET), which only the controller's account reaches. +func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) { + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}}) + q := Queue{Seat: seat} + api, err := jetstream.New(js.Conn()) + if err != nil { + return q, err + } + stream, err := api.Stream(ctx, worker.Stream) + if err != nil { + return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err) + } + consumer, err := stream.Consumer(ctx, worker.Name) + switch { + case errors.Is(err, jetstream.ErrConsumerNotFound): + // No holder has ever been given a worker: everything in the stream is waiting. + case err != nil: + return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err) + default: + info, err := consumer.Info(ctx) + if err != nil { + return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err) + } + q.Delivered = info.Delivered.Stream + q.AckPending = info.NumAckPending + q.MaxDeliver = info.Config.MaxDeliver + } + + // Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this + // is a handful of reads, never the history. + var asks []QueuedAsk + var seq uint64 = 1 + for n := 0; n < 10000; n++ { + msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat))) + if errors.Is(err, jetstream.ErrMsgNotFound) { + break + } + if err != nil { + return q, fmt.Errorf("reading the %s work queue: %w", seat, err) + } + ask := QueuedAsk{Seq: msg.Sequence} + if json.Unmarshal(msg.Data, &ask.Request) == nil { + r := ask.Request + ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref + if at, ok := BuildAskedAt(r.ID); ok { + ask.AskedAt = at + } + } + asks = append(asks, ask) + seq = msg.Sequence + 1 + } + + // What the events say, from shortly before the oldest ask behind the worker: its start, if any, + // came after it was asked. + var since time.Time + for _, a := range asks { + if a.Seq <= q.Delivered && (since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since))) { + since = a.AskedAt + } + } + heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}} + if len(asks) > 0 && q.Delivered > 0 { + if since.IsZero() { + since = time.Now().Add(-7 * 24 * time.Hour) + } + if heard, err = ReadBuildEvents(ctx, js, seat, since.Add(-time.Minute)); err != nil { + return q, err + } + } + q.Asks = ClassifyQueue(asks, q.Delivered, q.AckPending, heard) + return q, nil +} + +// ReadBuildEvents is every start and outcome the seat announced since a moment, from the events +// stream, with a consumer of its own that is gone when this returns. +func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, since time.Time) (BuildHeard, error) { + heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}} + // One token after `event`: started and built, never a build's log lines, which are two. + subject := "mesh.seat." + seat + ".event.*" + sub, err := js.Context().PullSubscribe(subject, "", + nats.BindStream(broker.EventsStream), nats.StartTime(since), nats.AckNone()) + if err != nil { + return heard, fmt.Errorf("cannot read %s from the bus: %w", subject, err) + } + defer func() { _ = sub.Unsubscribe() }() + for { + batch, err := sub.Fetch(500, nats.MaxWait(2*time.Second)) + if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) { + return heard, fmt.Errorf("reading %s: %w", subject, err) + } + for _, msg := range batch { + switch msg.Subject { + case BuildStartedOf(seat): + var s BuildStart + if json.Unmarshal(msg.Data, &s) == nil && s.ID != "" { + if was, ok := heard.Started[s.ID]; !ok || startedAt(s).After(startedAt(was)) { + heard.Started[s.ID] = s + } + } + case BuildOutcomeOf(seat): + var r BuildResult + if json.Unmarshal(msg.Data, &r) == nil && r.ID != "" { + heard.Outcomes[r.ID] = true + } + } + } + if len(batch) < 500 { + break + } + if ctx.Err() != nil { + return heard, ctx.Err() + } + } + return heard, nil +} + +// --- the cancelled set ---------------------------------------------------------------------- + +// safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern. +var safeKey = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// Cancelled is what the controller writes for an ask it cancelled. +type Cancelled struct { + At string `json:"at"` + Why string `json:"why"` +} + +// MarkCancelled puts an ask's id into the seat's cancelled set (novox/hq ADR 0219). The controller's +// act, before it deletes the ask from the queue. +func MarkCancelled(ctx context.Context, js *broker.JetStream, seat, id string) error { + if !safeKey.MatchString(id) { + return fmt.Errorf("%q is not a build id", id) + } + api, err := jetstream.New(js.Conn()) + if err != nil { + return err + } + kv, err := api.KeyValue(ctx, broker.CancelledSetName(seat)) + if err != nil { + return fmt.Errorf("the cancelled set of %s is not on the bus — the controller asserts it at its "+ + "start, so one older than this has not: %w", seat, err) + } + body, err := json.Marshal(Cancelled{At: time.Now().UTC().Format(time.RFC3339Nano), Why: CancelledByHand}) + if err != nil { + return err + } + _, err = kv.Put(ctx, id, body) + return err +} + +// UnmarkCancelled takes an ask's id out of the cancelled set: a cancel withdrawn, because the ask +// was taken as it was being cancelled. +func UnmarkCancelled(ctx context.Context, js *broker.JetStream, seat, id string) error { + if !safeKey.MatchString(id) { + return nil + } + api, err := jetstream.New(js.Conn()) + if err != nil { + return err + } + kv, err := api.KeyValue(ctx, broker.CancelledSetName(seat)) + if err != nil { + return err + } + return kv.Delete(ctx, id) +} + +// IsCancelled asks the seat's cancelled set whether an ask was cancelled: one direct read of one key, +// which is all a holder is granted of it. +func IsCancelled(conn *nats.Conn, seat, id string) (bool, error) { + if !safeKey.MatchString(id) { + // Not a key the controller could have written. + return false, nil + } + set := broker.CancelledSetName(seat) + reply, err := conn.Request("$JS.API.DIRECT.GET.KV_"+set+".$KV."+set+"."+id, nil, 3*time.Second) + if err != nil { + return false, err + } + return cancelledFrom(reply) +} + +// cancelledFrom reads a direct get's answer: a value is a cancel; not found, or a deletion marker, +// is not. +func cancelledFrom(reply *nats.Msg) (bool, error) { + if reply.Header != nil { + switch reply.Header.Get("Status") { + case "": + case "404": + return false, nil + default: + return false, fmt.Errorf("the cancelled set answered %s %s", + reply.Header.Get("Status"), reply.Header.Get("Description")) + } + if op := reply.Header.Get("KV-Operation"); op == "DEL" || op == "PURGE" { + return false, nil + } + } + return len(reply.Data) > 0, nil +} + +// --- a holder's verbs ----------------------------------------------------------------------- + +// NodeSeatToolSubject is where a node-scoped seat's verb is asked of one machine's holder (design 33 +// §4): the seat's verb subject with the machine as its last token. +func NodeSeatToolSubject(seat, verb, node string) string { + return SeatToolSubject(seat, verb) + "." + node +} + +// AskSeatTool asks one machine's holder of a node seat one of its verbs and reads its answer. +func AskSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string, args any, + timeout time.Duration) (Answer, error) { + body, err := json.Marshal(args) + if err != nil { + return Answer{}, err + } + asking, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + reply, err := conn.RequestWithContext(asking, NodeSeatToolSubject(seat, verb, node), body) + switch { + case errors.Is(err, nats.ErrNoResponders): + return Answer{}, fmt.Errorf("nothing on %s answers %s.%s: its holder is not running, or is "+ + "older than the verb", node, seat, verb) + case errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout): + return Answer{}, fmt.Errorf("%s did not answer %s.%s within %s", node, seat, verb, timeout) + case err != nil: + return Answer{}, err + } + var answer Answer + if err := json.Unmarshal(reply.Data, &answer); err != nil { + return Answer{}, fmt.Errorf("%s answered %s.%s with something unreadable: %w", node, seat, verb, err) + } + return answer, nil +} + +// --- a holder saying whether it takes work ---------------------------------------------------- + +// HolderState is what a holder says about itself whenever it is paused or resumed, at its start, +// and while it stays paused: whether it takes work (novox/hq ADR 0219). Retained on the events +// stream under the machine's own subject, so the last one is the machine's state. +type HolderState struct { + On string `json:"on"` + Paused bool `json:"paused"` + At string `json:"at"` +} + +// BuildPausedOf is where one machine's holder of a build seat says whether it takes work. +func BuildPausedOf(seat, node string) string { return "mesh.seat." + seat + ".event.paused." + node } + +// PausedSaid reads what each machine last said about taking work. A machine that never said is not +// in the answer — a holder older than ADR 0219 takes work and never pauses. +func PausedSaid(js *broker.JetStream, seat string, nodes []string) (map[string]HolderState, error) { + out := map[string]HolderState{} + for _, node := range nodes { + msg, err := js.Context().GetLastMsg(broker.EventsStream, BuildPausedOf(seat, node)) + if errors.Is(err, nats.ErrMsgNotFound) { + continue + } + if err != nil { + return out, err + } + var s HolderState + if json.Unmarshal(msg.Data, &s) == nil { + out[node] = s + } + } + return out, nil +} diff --git a/internal/link/queue_test.go b/internal/link/queue_test.go new file mode 100644 index 0000000..4ff3a74 --- /dev/null +++ b/internal/link/queue_test.go @@ -0,0 +1,336 @@ +package link + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The build queue read and changed by hand (novox/hq ADR 0219). + +// Which ask is waiting, in flight and dead, from the stream, the worker and the seat's events. +func TestTheQueueTellsWaitingInFlightAndDeadApart(t *testing.T) { + at := func(m int) string { + return time.Date(2026, 10, 5, 12, m, 0, 0, time.UTC).Format(time.RFC3339Nano) + } + asks := []QueuedAsk{ + {Seq: 1, ID: "dead-1"}, // handed out five times, its machine moved on: dead + {Seq: 2, ID: "running-g"}, // g14's latest start, no outcome: in flight + {Seq: 3, ID: "running-a"}, // ace's latest start, no outcome: in flight + {Seq: 4, ID: "unsaid"}, // taken, not yet said: in flight while the worker counts it + {Seq: 5, ID: "waiting-1"}, // after the worker's last delivery + {Seq: 6, ID: "waiting-2"}, + } + heard := BuildHeard{ + Started: map[string]BuildStart{ + "dead-1": {ID: "dead-1", On: "g14", At: at(1)}, + "running-g": {ID: "running-g", On: "g14", At: at(5)}, + "running-a": {ID: "running-a", On: "ace", At: at(6)}, + "done": {ID: "done", On: "novox", At: at(7)}, + }, + Outcomes: map[string]bool{"done": true}, + } + got := ClassifyQueue(asks, 4, 2, heard) + want := map[string]string{"dead-1": AskDead, "running-g": AskInFlight, "running-a": AskInFlight, + "unsaid": AskDead, "waiting-1": AskWaiting, "waiting-2": AskWaiting} + for _, a := range got { + if a.State != want[a.ID] { + t.Errorf("%s is %s, want %s", a.ID, a.State, want[a.ID]) + } + } + q := Queue{Asks: got} + if a, _ := q.Find("running-a"); a.On != "ace" || a.Started.IsZero() { + t.Errorf("an ask in flight does not say where: %+v", a) + } + + // **The worker's count bounds it**: with one pending, the machine whose start is oldest died + // with its ask. + got = ClassifyQueue(asks, 4, 1, heard) + q = Queue{Asks: got} + if a, _ := q.Find("running-a"); a.State != AskInFlight { + t.Errorf("running-a is %s", a.State) + } + if a, _ := q.Find("running-g"); a.State != AskDead || a.On != "" { + t.Errorf("past the worker's count running-g is %s on %q", a.State, a.On) + } + // **Handed back after a holder restarted mid-build**: started on g14, g14 then started something + // else, and the worker still counts it — it waits to be handed out again, and is not dead. With + // no more asks behind the worker than it counts pending, none is dead. + restarted := []QueuedAsk{{Seq: 1, ID: "dead-1"}, {Seq: 2, ID: "running-g"}} + for _, a := range ClassifyQueue(restarted, 2, 2, heard) { + if a.State != AskInFlight { + t.Errorf("%s is %s with two behind the worker and two pending", a.ID, a.State) + } + if a.ID == "dead-1" && (a.On != "" || a.Was != "g14") { + t.Errorf("an ask handed back reads on %q, was on %q", a.On, a.Was) + } + } + // The handed-back one takes a leftover slot before an ask nobody said started. + got = ClassifyQueue(asks, 4, 4, heard) + q = Queue{Asks: got} + for _, id := range []string{"dead-1", "running-g", "running-a", "unsaid"} { + if a, _ := q.Find(id); a.State != AskInFlight { + t.Errorf("with four pending %s is %s", id, a.State) + } + } + got = ClassifyQueue(asks, 4, 3, heard) + q = Queue{Asks: got} + if a, _ := q.Find("dead-1"); a.State != AskInFlight { + t.Errorf("the handed-back ask lost its slot to one nobody said: %s", a.State) + } + if a, _ := q.Find("unsaid"); a.State != AskDead { + t.Errorf("unsaid is %s", a.State) + } + + // None pending: everything behind the worker is dead, and names no machine. + for _, a := range ClassifyQueue(asks, 4, 0, heard) { + if a.Seq <= 4 && (a.State != AskDead || a.On != "") { + t.Errorf("%s with nothing pending is %s on %q", a.ID, a.State, a.On) + } + } +} + +// What a direct read of the cancelled set answers. +func TestACancelledSetReadSaysWhatWasCancelled(t *testing.T) { + found := &nats.Msg{Header: nats.Header{"Nats-Subject": []string{"$KV.x.build-1"}}, Data: []byte(`{"why":"cancelled by hand"}`)} + if c, err := cancelledFrom(found); err != nil || !c { + t.Errorf("a value read as %v %v", c, err) + } + missing := &nats.Msg{Header: nats.Header{"Status": []string{"404"}}} + if c, err := cancelledFrom(missing); err != nil || c { + t.Errorf("not found read as %v %v", c, err) + } + deleted := &nats.Msg{Header: nats.Header{"KV-Operation": []string{"DEL"}}} + if c, err := cancelledFrom(deleted); err != nil || c { + t.Errorf("a deletion marker read as %v %v", c, err) + } + broken := &nats.Msg{Header: nats.Header{"Status": []string{"408"}, "Description": []string{"Request Timeout"}}} + if _, err := cancelledFrom(broken); err == nil { + t.Error("an error answer read as an answer") + } + if c, err := IsCancelled(nil, TheBuildMachine, "a.b.>"); err != nil || c { + t.Error("a pattern was looked up as a key") + } +} + +// --- against a real server ---------------------------------------------------------------------- + +func aBusWithACancelledSet(t *testing.T) *broker.JetStream { + t.Helper() + js := aBusWithTheBuildRole(t) + seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}} + if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) }) + return js +} + +// **The race a delete alone leaves open**: an ask fetched in the moment it was cancelled is ended by +// the holder that took it — terminated, never built, its outcome said as failed. +func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) { + js := aBusWithACancelledSet(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil { + t.Fatal(err) + } + outcomes, err := js.Conn().SubscribeSync(BuildOutcome()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = outcomes.Unsubscribe() }() + _ = js.Conn().Flush() + for _, id := range []string{"build-cancelled", "build-kept"} { + body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + } + + built := make(chan string, 2) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + built <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case id := <-built: + if id != "build-kept" { + t.Fatalf("%s was built", id) + } + case <-time.After(10 * time.Second): + t.Fatal("the ask that was not cancelled was never built") + } + msg, err := outcomes.NextMsg(5 * time.Second) + if err != nil { + t.Fatal(err) + } + var said BuildResult + if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" || + said.Failed != CancelledByHand || said.On != "anchor" { + t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the cancelled ask is still queued: terminated, it would not be") +} + +// A paused holder takes nothing; resumed, it takes what waited. +func TestNatsAPausedHolderTakesNothingNew(t *testing.T) { + js := aBusWithACancelledSet(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + paused := make(chan bool, 1) + paused <- true + isPaused := func() bool { + p := <-paused + paused <- p + return p + } + took := make(chan string, 1) + machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused}) + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + t.Fatalf("a paused holder took %s", id) + case <-time.After(3 * time.Second): + } + q, err := ReadQueue(ctx, js, TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting { + t.Fatalf("while paused the queue reads %+v", q.Asks) + } + <-paused + paused <- false + select { + case id := <-took: + if id != "build-waits" { + t.Fatalf("took %s", id) + } + case <-time.After(10 * time.Second): + t.Fatal("resumed, the holder never took what waited") + } +} + +// What a holder last said about taking work is read back per machine. +func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) { + js := aBusWithTheBuildRole(t) + for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} { + body, _ := json.Marshal(s) + if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil { + t.Fatal(err) + } + } + said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"}) + if err != nil { + t.Fatal(err) + } + if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 { + t.Fatalf("read back %+v", said) + } +} + +// A `build` waiting on its outcome hears a cancel — which publishes no outcome — from the cancelled +// set, and a kill from the outcome the holder announces (novox/hq ADR 0219). +func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) { + js := aBusWithACancelledSet(t) + was := cancelLook + cancelLook = 200 * time.Millisecond + t.Cleanup(func() { cancelLook = was }) + ask := &natsBuilds{js: js, seat: TheBuildMachine} + + go func() { + time.Sleep(500 * time.Millisecond) + _ = MarkCancelled(context.Background(), js, TheBuildMachine, "build-waited-cancelled") + }() + result, err := ask.Submit(context.Background(), BuildRequest{ID: "build-waited-cancelled", Repository: "/r"}, 10*time.Second) + if err != nil || result.Failed != CancelledByHand || result.ID != "build-waited-cancelled" { + t.Fatalf("the waiter heard %+v (%v)", result, err) + } + + // Killed: the holder announces the failure as any outcome, and the waiter has it. + ctx, stop := context.WithCancel(context.Background()) + defer stop() + machine := MachineOverNATS(js, "ace") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + r := work.Request() + _ = work.Announce(ctx, BuildResult{ID: r.ID, Repository: r.Repository, On: "ace", Failed: KilledByHand}) + _ = work.Done() + }) + }() + result, err = ask.Submit(context.Background(), BuildRequest{ID: "build-waited-killed", Repository: "/r"}, 10*time.Second) + if err != nil || result.Failed != KilledByHand || result.On != "ace" { + t.Fatalf("the waiter heard %+v (%v)", result, err) + } +} + +// Paused in the moment a pull was answered: the ask is handed back, not built (ADR 0219). +func TestNatsAHolderPausedAsItsPullWasAnsweredHandsTheAskBack(t *testing.T) { + js := aBusWithACancelledSet(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + // Not paused when it asks; paused by the time the answer is read. + var looks int + var mu sync.Mutex + paused := func() bool { + mu.Lock() + defer mu.Unlock() + looks++ + return looks > 1 + } + took := make(chan string, 1) + machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: paused}) + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID }) + }() + time.Sleep(200 * time.Millisecond) + body, _ := json.Marshal(BuildRequest{ID: "build-handed-back", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + t.Fatalf("a holder paused as its pull was answered built %s", id) + case <-time.After(2 * time.Second): + } + q, err := ReadQueue(ctx, js, TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].ID != "build-handed-back" { + t.Fatalf("the ask is not back in the queue: %+v", q.Asks) + } +} diff --git a/internal/link/seattools.go b/internal/link/seattools.go index f579c53..89b24a5 100644 --- a/internal/link/seattools.go +++ b/internal/link/seattools.go @@ -42,6 +42,18 @@ const RebindAfter = 30 * time.Second // tried again (2026-09-30). A refused subscription is therefore retried until it holds: the server // says so asynchronously and invalidates the subscription, which is what is checked. func (b OverNATS) ServeSeatTools(seat string, handlers map[string]ToolHandler, logger *log.Logger) (func(), error) { + return b.serveTools(seat, func(verb string) string { return SeatToolSubject(seat, verb) }, handlers, logger) +} + +// ServeNodeSeatTools is ServeSeatTools for one machine's holder of a node-scoped seat (novox/hq ADR +// 0159, ADR 0219): each verb on the seat's subject for this machine and no other, so a call names +// the machine it is for and only that machine's holder answers it. +func (b OverNATS) ServeNodeSeatTools(seat, node string, handlers map[string]ToolHandler, logger *log.Logger) (func(), error) { + return b.serveTools(seat, func(verb string) string { return NodeSeatToolSubject(seat, verb, node) }, handlers, logger) +} + +func (b OverNATS) serveTools(seat string, subjectOf func(string) string, handlers map[string]ToolHandler, + logger *log.Logger) (func(), error) { var subs []*nats.Subscription done := make(chan struct{}) stop := func() { @@ -52,7 +64,7 @@ func (b OverNATS) ServeSeatTools(seat string, handlers map[string]ToolHandler, l } for verb, handle := range handlers { verb, handle := verb, handle - subject := SeatToolSubject(seat, verb) + subject := subjectOf(verb) bind := func() (*nats.Subscription, error) { return b.Conn.QueueSubscribe(subject, "seat."+seat, func(msg *nats.Msg) { // Its own goroutine per call: a slow `push` must not hold up a `status` asked beside it,