On the control node the store-bound packages of the controller's suite ran five to seven times slower than on any other holder, and every controller check that landed there ran past the suite's thirty minutes: a throwaway store flushing to a disk the mesh's own store, bus and forge keep busy. A store that lives for one check needs no crash safety. A holder recreated mid-check left the check's store and bus running, and the redelivery went to another machine, so nothing removed them: eleven pairs across four machines. A starting holder has taken nothing, so every container labelled with an ask of the seat is an earlier holder's.
173 lines
6.7 KiB
Go
173 lines
6.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
|
|
// 0219), and what it says about itself follows.
|
|
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
|
|
workspace := t.TempDir()
|
|
var said []link.HolderState
|
|
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
|
|
said = append(said, s)
|
|
return nil
|
|
})
|
|
if h.Paused() {
|
|
t.Fatal("a new holder starts paused")
|
|
}
|
|
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
|
|
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
|
|
}
|
|
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
|
|
if !again.Paused() {
|
|
t.Fatal("restarted, the holder forgot it was paused")
|
|
}
|
|
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
|
|
t.Fatal("resumed, the holder came back paused")
|
|
}
|
|
}
|
|
|
|
// kill ends the build with that id — its context, which ends its commands — removes what it left by
|
|
// label, and refuses an id it is not building.
|
|
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
|
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
|
var removed []string
|
|
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
|
removed = append(removed, name+" "+strings.Join(args, " "))
|
|
if args[0] == "ps" {
|
|
return "c1\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
kill := h.handlers()["kill"]
|
|
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
|
|
t.Fatal("killed a build while none ran")
|
|
}
|
|
|
|
building, cancel := context.WithCancel(context.Background())
|
|
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
|
|
h.stepped(r, "image")
|
|
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
|
|
t.Fatalf("current says %+v", c)
|
|
}
|
|
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
|
|
!strings.Contains(err.Error(), "build-1") {
|
|
t.Fatalf("killed another id: %v", err)
|
|
}
|
|
// The build's own goroutine: it ends when its context does, as a build's commands do, and
|
|
// announces what came of it.
|
|
go func() {
|
|
<-building.Done()
|
|
if h.returned(r) {
|
|
h.said(r, link.KilledByHand, true)
|
|
}
|
|
h.end(r)
|
|
}()
|
|
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if building.Err() == nil {
|
|
t.Fatal("the build's context was not cancelled")
|
|
}
|
|
if !r.killed {
|
|
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
|
|
}
|
|
said := answer.(map[string]any)["said"].(string)
|
|
if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
|
|
t.Errorf("kill said %q", said)
|
|
}
|
|
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
|
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
|
|
t.Errorf("removed %v", removed)
|
|
}
|
|
if c := h.current(); c.Running != nil {
|
|
t.Errorf("after the kill current says %+v", c)
|
|
}
|
|
}
|
|
|
|
// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
|
|
func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
|
|
info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
|
|
if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
|
|
t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
|
|
}
|
|
kill := info.Endpoints[1]
|
|
md := kill.Metadata
|
|
if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
|
|
md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
|
|
!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
|
|
t.Fatalf("kill is announced as %+v", kill)
|
|
}
|
|
body, err := json.Marshal(info)
|
|
if err != nil || len(body) > 8*1024 {
|
|
t.Fatalf("the answer is %d bytes (%v)", len(body), err)
|
|
}
|
|
}
|
|
|
|
// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
|
|
// own error is its outcome. One whose outcome could not be announced is not said to be settled.
|
|
func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
|
|
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
|
h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
|
|
_, cancel := context.WithCancel(context.Background())
|
|
r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
|
|
if killed := h.returned(r); killed {
|
|
t.Fatal("a build nobody killed reads as killed")
|
|
}
|
|
if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
|
|
t.Fatalf("killed a build that had ended: %v", err)
|
|
}
|
|
h.end(r)
|
|
|
|
building, cancel := context.WithCancel(context.Background())
|
|
r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
|
|
go func() {
|
|
<-building.Done()
|
|
if h.returned(r) {
|
|
h.said(r, link.KilledByHand, false)
|
|
}
|
|
h.end(r)
|
|
}()
|
|
said, err := h.kill("build-2")
|
|
if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
|
|
t.Fatalf("kill said %q (%v)", said, err)
|
|
}
|
|
}
|
|
|
|
// **Issue 306**: a holder starting removes the containers earlier builds left on its machine, through
|
|
// the runner a kill uses, and a runtime that cannot be asked does not stop it.
|
|
func TestAHolderStartingRemovesWhatEarlierBuildsLeftHere(t *testing.T) {
|
|
h := newHolder("novox", link.TheBuildMachine, t.TempDir(), nil)
|
|
var removed []string
|
|
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
|
removed = append(removed, name+" "+strings.Join(args, " "))
|
|
if args[0] == "ps" {
|
|
return "c1 build-1\nc2 build-1\nc3 check-here-2\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
if n := h.removeLeftBehind(); n != 2 || len(removed) != 2 || removed[1] != "docker rm -f c1 c2" {
|
|
t.Fatalf("removed %d: %v", n, removed)
|
|
}
|
|
h.remove = func(context.Context, string, string, ...string) (string, error) {
|
|
return "", errors.New("no runtime")
|
|
}
|
|
if n := h.removeLeftBehind(); n != 0 {
|
|
t.Errorf("a runtime that cannot be asked removed %d", n)
|
|
}
|
|
}
|