diff --git a/.gitignore b/.gitignore index ff196481..3f98af11 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ # A built binary. The lab writes one here when pointed at the repository root by mistake; # built artifacts belong in build/, which is already ignored. -/mesh-builder /mesh-controller /postgres-provisioner /redis-provisioner diff --git a/Makefile b/Makefile index 8885d70c..721bb524 100644 --- a/Makefile +++ b/Makefile @@ -46,17 +46,6 @@ image: @echo @docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' -# The builder ships as an image too, because it is a module the mesh assigns rather than a program -# somebody starts on a machine by hand. -BUILDER_IMAGE ?= mesh-builder:$(VERSION) -BUILDER_DEV_TAG ?= mesh-builder:development - -builder-image: - @test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE="; exit 1; } - docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) . - @echo - @docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' - # The provisioner ships as an image too, because it is the thing that makes a sealed credential # true on a machine -- and the mesh cannot, having discarded the plaintext. PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION) diff --git a/cmd/mesh-builder/Dockerfile b/cmd/mesh-builder/Dockerfile deleted file mode 100644 index b639b869..00000000 --- a/cmd/mesh-builder/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -ARG ALPINE_BASE=alpine:3 -ARG GO_BASE=golang:1.25-alpine -# The builder, as a module ships one. -# -# Not FROM scratch, unlike the control plane: this one runs git and a container client, so it -# needs a filesystem with them on it. That is the honest cost of a machine whose job is to build — -# and it is why building is a MODULE on a machine that has a runtime rather than something the -# control plane does (novox/hq ADR 0005). -FROM ${GO_BASE} AS build -WORKDIR /src -COPY go.mod go.sum ./ -RUN go mod download -COPY . . -RUN CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags '-s -w' -o /mesh-builder ./cmd/mesh-builder - -FROM ${ALPINE_BASE} -# git to clone what it is asked to build, and the docker client to build and push it. The daemon -# is the machine's, reached through its socket — a build machine shares the runtime it was given -# rather than running one inside itself. -RUN apk add --no-cache git docker-cli -COPY --from=build /mesh-builder /usr/local/bin/mesh-builder -ENTRYPOINT ["/usr/local/bin/mesh-builder"] diff --git a/cmd/mesh-builder/busy.go b/cmd/mesh-builder/busy.go deleted file mode 100644 index 049e97b2..00000000 --- a/cmd/mesh-builder/busy.go +++ /dev/null @@ -1,235 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/novox/mesh-controller/internal/builder" -) - -// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its -// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it -// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes -// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check -// it runs, instead of the build queue handing it out again from the start (issue 594). -// -// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts -// without them, and a restarted one removes them before it takes work again. -// -// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed -// when the build ends. -// - drained: written by the ask, before it reads whether a build is in hand. -// -// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand, -// or the agent sees the drain and hands the request back; never neither. - -func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") } -func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") } - -// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it -// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped — -// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer -// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never -// came, which is a fault to see. -const ( - drainLapse = 2 * time.Minute - drainTooLong = 15*time.Minute + 5*time.Minute -) - -// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark. -func drainedAt() (began, asked time.Time, ok bool) { - info, err := os.Stat(drainMark()) - if err != nil { - return time.Time{}, time.Time{}, false - } - asked = info.ModTime() - began = asked - if raw, err := os.ReadFile(drainMark()); err == nil { - if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) { - began = t - } - } - return began, asked, true -} - -// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request. -func drained() bool { return drainedSince(time.Now()) } - -func drainedSince(now time.Time) bool { - _, asked, ok := drainedAt() - return ok && now.Sub(asked) < drainLapse -} - -// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for -// longer than its node-engine's bound and the slack. -func drainedTooLong(now time.Time) error { - began, asked, ok := drainedAt() - if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong { - return nil - } - return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+ - "within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong) -} - -// inHand is true while this process has claimed a build and not yet released it. The mark in the temporary directory -// is only believed while it is: a mark with no build of this process behind it is stale (a crash, a fault in an -// earlier path) and is removed, because left alone it hands back every ask for ever (issue 1768). -var inHand atomic.Bool - -// markRetry is how long an agent that cannot mark a build in hand takes no work, and markFault when it last failed to. -// Taking nothing is the point: an ask fetched and handed back at once, many times a second, spends all its deliveries. -const markRetry = 30 * time.Second - -var ( - markMu sync.Mutex - markFault time.Time - markSaid string -) - -// markFaulty says the agent could not mark a build in hand and has not yet been able to since: it takes no work. Once -// markRetry has passed it tries the mark itself, with no ask in hand, so that an ask is fetched only when marking works. -func markFaulty(now time.Time) bool { - markMu.Lock() - failed := markFault - markMu.Unlock() - if failed.IsZero() { - return false - } - if now.Sub(failed) < markRetry { - return true - } - f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) - if err != nil && errors.Is(err, os.ErrExist) && !inHand.Load() { - _ = os.Remove(inHandMark()) - f, err = os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) - } - if err != nil { - markFailed(err) - return true - } - _ = f.Close() - _ = os.Remove(inHandMark()) - markMu.Lock() - markFault, markSaid = time.Time{}, "" - markMu.Unlock() - fmt.Fprintf(os.Stderr, "the mark of a build in hand can be made again (%s): taking builds\n", inHandMark()) - return false -} - -// markFailed notes that a build could not be marked in hand, and says it when it is a new reason. -func markFailed(err error) { - markMu.Lock() - defer markMu.Unlock() - markFault = time.Now() - if said := err.Error(); said != markSaid { - markSaid = said - fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so this agent takes no build for %s and tries again: %v\n", - inHandMark(), markRetry, err) - } -} - -// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes -// back to the queue. -func claim() bool { - if !inHand.Load() { - // Nothing of this process is in hand, so a mark already there is stale. - if err := os.Remove(inHandMark()); err == nil { - fmt.Fprintf(os.Stderr, "removed the mark of a build in hand (%s): no build of this agent is in hand\n", inHandMark()) - } - } - // Made, never written over: one build at a time, and a mark already there is a build in hand. - f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) - if err != nil { - // Unmarked, an ask would say idle while this build runs: the build is not taken. - markFailed(err) - return false - } - _, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n") - if closeErr := f.Close(); err == nil { - err = closeErr - } - if err != nil { - _ = os.Remove(inHandMark()) - markFailed(err) - return false - } - if drained() { - _ = os.Remove(inHandMark()) - fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back") - return false - } - inHand.Store(true) - return true -} - -// holding says what is in hand, for the ask to print: written once the request is read. -func holding(what string) { - _ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644) -} - -// released ends the build in hand. -func released() { - inHand.Store(false) - if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) { - fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err) - } -} - -// undrained clears both marks at the start of a holder: a container started again is asked afresh. -func undrained() error { - for _, mark := range []string{drainMark(), inHandMark()} { - if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("cannot remove %s: %w", mark, err) - } - } - return nil -} - -// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is. -func busy() (string, error) { - // The mark keeps when the drain began while it is asked within drainLapse, and is written again on every - // ask: its time is the last ask. - began := time.Now().UTC() - if was, _, ok := drainedAt(); ok && drained() { - began = was.UTC() - } - if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil { - // Not drained, it could take the next build: said, and answered busy only for what is in hand now. - fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err) - } - held, err := os.ReadFile(inHandMark()) - if err != nil { - return "", errors.New("idle: no build in hand, and this agent takes no new one") - } - since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ") - if what == "" { - what = "a build" - } - return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil -} - -var ( - roomMu sync.Mutex - roomSaid bool -) - -// noRoom says this node is under the floor of free space (novox/hq issue 1834): it takes no work, so another build node -// takes the ask instead of this one taking it and ending it in an error. Said once when it starts and again when it ends. -func noRoom(workspace string) bool { - err := builder.RoomToWork("/", workspace) - roomMu.Lock() - defer roomMu.Unlock() - if err != nil && !roomSaid { - fmt.Fprintf(os.Stderr, "taking no builds: %v\n", err) - } else if err == nil && roomSaid { - fmt.Fprintf(os.Stderr, "there is room again: taking builds\n") - } - roomSaid = err != nil - return err != nil -} diff --git a/cmd/mesh-builder/busy_test.go b/cmd/mesh-builder/busy_test.go deleted file mode 100644 index d5f89b7f..00000000 --- a/cmd/mesh-builder/busy_test.go +++ /dev/null @@ -1,239 +0,0 @@ -package main - -import ( - "github.com/novox/mesh-controller/internal/builder" - "os" - "strings" - "sync" - "testing" - "time" -) - -// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the -// agent takes no new build request (novox/hq ADR 0335 §5). -func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) { - t.Setenv("TMPDIR", t.TempDir()) - if err := undrained(); err != nil { - t.Fatal(err) - } - if drained() { - t.Fatal("an agent never asked is drained") - } - // A build is claimed and runs. - if !claim() { - t.Fatal("an agent never asked refused a build") - } - holding("novox/mesh-controller (check-280)") - said, err := busy() - if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") { - t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err) - } - // From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back. - if !drained() { - t.Fatal("the ask did not drain the agent") - } - if claim() { - t.Fatal("a drained agent took a new build") - } - // The build in hand is still in hand after the refused claim, and busy goes on saying so. - if _, err := busy(); err != nil { - t.Fatalf("a refused claim ended the build in hand: %v", err) - } - // It ends: idle, and still drained. - released() - if _, err := busy(); err == nil { - t.Fatal("an agent with nothing in hand answered busy") - } - if claim() { - t.Fatal("a drained agent took a build after its last one ended") - } - // Started again, it is asked afresh. - if err := undrained(); err != nil { - t.Fatal(err) - } - if drained() || !claim() { - t.Fatal("a holder started again is still drained") - } - released() -} - -// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never -// neither, which would replace the container under a build just taken. -func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) { - t.Setenv("TMPDIR", t.TempDir()) - for i := 0; i < 200; i++ { - if err := undrained(); err != nil { - t.Fatal(err) - } - var claimed bool - var askErr error - var wg sync.WaitGroup - wg.Add(2) - go func() { defer wg.Done(); claimed = claim() }() - go func() { defer wg.Done(); _, askErr = busy() }() - wg.Wait() - if claimed && askErr != nil { - t.Fatalf("round %d: the agent took a build and the ask said idle", i) - } - released() - } -} - -// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a -// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green. -func TestADrainNoLongerAskedLapses(t *testing.T) { - t.Setenv("TMPDIR", t.TempDir()) - if err := undrained(); err != nil { - t.Fatal(err) - } - if _, err := busy(); err == nil { - t.Fatal("an idle agent answered busy") - } - if !drained() || claim() { - t.Fatal("an agent just asked takes work") - } - // The last ask was three minutes ago. - old := time.Now().Add(-3 * time.Minute) - if err := os.Chtimes(drainMark(), old, old); err != nil { - t.Fatal(err) - } - if drained() { - t.Fatal("a drain no longer asked for three minutes still holds") - } - if !claim() { - t.Fatal("an agent whose drain lapsed takes no work") - } - released() -} - -// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the -// replacement it was drained for never came. One asked again after a lapse starts a new drain. -func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) { - t.Setenv("TMPDIR", t.TempDir()) - if err := undrained(); err != nil { - t.Fatal(err) - } - taking("mesh-build-machine") - if _, err := busy(); err == nil { - t.Fatal("an idle agent answered busy") - } - now := time.Now() - if err := drainedTooLong(now); err != nil { - t.Fatalf("an agent drained a moment ago is not ready: %v", err) - } - // The drain began 25 minutes ago, and the node-engine asked a moment ago. - began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339) - if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil { - t.Fatal(err) - } - err := drainedTooLong(time.Now()) - if err == nil || !strings.Contains(err.Error(), "has not come") { - t.Fatalf("an agent drained for 25 minutes reads ready: %v", err) - } - // Each ask keeps when the drain began. - if _, err := busy(); err == nil { - t.Fatal("an idle agent answered busy") - } - if err := drainedTooLong(time.Now()); err == nil { - t.Fatal("an ask started the drain again") - } - // The asks stopped: the drain lapsed, and so did the fault. - old := time.Now().Add(-3 * time.Minute) - if err := os.Chtimes(drainMark(), old, old); err != nil { - t.Fatal(err) - } - if err := drainedTooLong(time.Now()); err != nil { - t.Fatalf("a lapsed drain still reads not ready: %v", err) - } -} - -// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the -// node-engine's whole bound. -func TestAnUnknownCommandFails(t *testing.T) { - was := os.Args - t.Cleanup(func() { os.Args = was }) - os.Args = []string{"mesh-builder", "some-later-word"} - if err := run(); err == nil { - t.Fatal("an unknown command exited 0") - } - os.Args = []string{"mesh-builder", "help"} - if err := run(); err != nil { - t.Fatalf("help failed: %v", err) - } -} - -// 1768/AC2: a mark naming no build of this agent (left by a crash or a fault) is stale: the next ask removes it and is -// run, never fetched and handed back in a loop. -func TestAStaleInHandMarkIsRemovedAndTheAskRuns(t *testing.T) { - t.Setenv("TMPDIR", t.TempDir()) - inHand.Store(false) - if err := undrained(); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(inHandMark(), []byte("2026-10-21T18:45:06Z\n"), 0o644); err != nil { - t.Fatal(err) - } - if !claim() { - t.Fatal("a stale in-hand mark made the agent hand back an ask") - } - // A live one — this agent's own build in hand — is never taken over. - if claim() { - t.Fatal("a second build was taken while one is in hand") - } - if _, err := busy(); err != nil { - t.Fatalf("the build in hand was lost: %v", err) - } - released() -} - -// 1768/AC3: an agent that cannot mark a build in hand for another reason takes no work, instead of fetching asks and -// handing them back; it takes work again once marking works. -func TestAnAgentThatCannotMarkABuildTakesNoWorkUntilItCan(t *testing.T) { - dir := t.TempDir() - t.Setenv("TMPDIR", dir) - inHand.Store(false) - markFault, markSaid = time.Time{}, "" - t.Cleanup(func() { markFault, markSaid = time.Time{}, "" }) - if err := undrained(); err != nil { - t.Fatal(err) - } - // The mark's place is a directory that cannot be removed or made over: a directory named as the mark. - if err := os.MkdirAll(inHandMark()+"/x", 0o755); err != nil { - t.Fatal(err) - } - if claim() { - t.Fatal("an agent that cannot mark a build took it") - } - if !markFaulty(time.Now()) { - t.Fatal("an agent that cannot mark a build still takes work") - } - // Still failing after the retry time: still takes nothing, and no ask was needed to find out. - if !markFaulty(time.Now().Add(2 * markRetry)) { - t.Fatal("an agent whose mark still cannot be made takes work") - } - if err := os.RemoveAll(inHandMark()); err != nil { - t.Fatal(err) - } - if markFaulty(time.Now().Add(4 * markRetry)) { - t.Fatal("an agent whose mark can be made again takes no work") - } - if !claim() { - t.Fatal("an agent that can mark takes no build") - } - released() -} - -// 1834/AC4: given a node under the floor of free space, when the taking loop asks, then it takes nothing, and takes again -// once there is room. -func TestANodeUnderTheFloorOfFreeSpaceTakesNoWork(t *testing.T) { - was := builder.FreeBytes - t.Cleanup(func() { builder.FreeBytes = was }) - builder.FreeBytes = func(string) (uint64, error) { return 2 << 30, nil } - if !noRoom(t.TempDir()) { - t.Error("a node with 2 GiB free took work") - } - builder.FreeBytes = func(string) (uint64, error) { return 200 << 30, nil } - if noRoom(t.TempDir()) { - t.Error("a node with 200 GiB free refused work") - } -} diff --git a/cmd/mesh-builder/check_spec_test.go b/cmd/mesh-builder/check_spec_test.go deleted file mode 100644 index 808c32b9..00000000 --- a/cmd/mesh-builder/check_spec_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package main - -import ( - "testing" - - "github.com/novox/mesh-controller/internal/builder" - "github.com/novox/mesh-controller/internal/link" -) - -// The build seat hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own. -func TestTheBuildSeatHandsTheCheckTheJudgeCacheItDeclares(t *testing.T) { - t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache") - if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" { - t.Fatalf("the judge cache the manifest declares is read as %q", got) - } - t.Setenv("MESH_JUDGE_CACHE", "") - if got := judgeCache(); got != "" { - t.Fatalf("with no judge cache declared, %q is used", got) - } -} - -// The siblings a check is asked to read reach the build seat's spec and, as read, the outcome (novox/hq ADR 0382). -func TestTheSiblingsAnAskNamesReachTheSpecAndTheOutcome(t *testing.T) { - spec := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main", - Siblings: []link.CheckSibling{{Repository: "novox/mesh-controller", Number: 346, Head: "a", Base: "main", Dir: "mesh-controller"}}, - SiblingsUnknown: "why"}}) - if len(spec.Siblings) != 1 || spec.Siblings[0] != (builder.Sibling{Repository: "novox/mesh-controller", Number: 346, - Head: "a", Base: "main", Dir: "mesh-controller"}) || spec.SiblingsUnknown != "why" { - t.Fatalf("the spec reads %+v", spec) - } - out := siblingsOut([]builder.JudgedSibling{{Repository: "novox/mesh-controller", Number: 346, Head: "a", Base: "main", BaseCommit: "b"}}) - if len(out) != 1 || out[0].BaseCommit != "b" || out[0].Number != 346 { - t.Fatalf("the outcome reads %+v", out) - } - if plain := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "o", Repo: "r"}}); plain.Siblings != nil { - t.Fatalf("a request that names none has siblings %+v", plain.Siblings) - } -} - -// 1826/AC2: the held values a check request carries reach the spec; a request that carries none leaves it empty. -func TestTheHeldValuesAnAskCarriesReachTheSpec(t *testing.T) { - spec := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", - HeldConfig: []link.HeldConfig{{Pull: "novox/mesh-catalog#434", Head: "a", Node: "ace", Module: "n8n", Key: "media-gid", - Value: []byte("2000"), Approved: "the operator"}}}}) - if len(spec.Held) != 1 || spec.Held[0].Node != "ace" || spec.Held[0].Key != "media-gid" || string(spec.Held[0].Value) != "2000" || - spec.Held[0].Approved != "the operator" { - t.Fatalf("the spec reads %+v", spec.Held) - } - if plain := checkSpecOf(link.BuildRequest{Check: &link.CheckRequest{Owner: "o", Repo: "r"}}); plain.Held != nil { - t.Fatalf("a request that carries none has %+v", plain.Held) - } -} diff --git a/cmd/mesh-builder/holder.go b/cmd/mesh-builder/holder.go deleted file mode 100644 index ff962c44..00000000 --- a/cmd/mesh-builder/holder.go +++ /dev/null @@ -1,433 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/micro" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/builder" - "github.com/novox/mesh-controller/internal/link" -) - -// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219). -// -// **The queue is the controller's; the build running here is this machine's.** The controller can -// see and change what waits in the seat's queue, but an ask a holder already took is a process tree -// on this machine and containers in this machine's runtime, and only this machine can end them. So -// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill -// it, pause, resume. -// -// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global -// set per build — so "the build running here" is one or none, and kill names it by id so a call -// that arrives as one build ends and the next begins cannot end the wrong one. - -// holder is this machine's state as a holder of the build seat. -type holder struct { - on, seat string - // workspace is where the paused flag is kept, so a holder restarted while paused stays paused - // rather than silently taking work again. - workspace string - // say publishes this machine's state: whether it takes work (link.HolderState). - say func(link.HolderState) error - // remove runs what a kill needs outside the build: the containers left behind. - remove builder.Runner - // scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where - // this machine publishes to no artifact store. - scratch builder.Scratch - - mu sync.Mutex - paused bool - running *running -} - -// running is the build this machine is doing. -type running struct { - request link.BuildRequest - step string - started time.Time - cancel context.CancelFunc - killed bool - // returned is the build's own work having ended, before a kill or not; outcome is what was then - // announced, and announced whether it went out. - returned bool - outcome string - announced bool - done chan struct{} -} - -// pausedFile is where the flag lives in the workspace. -func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") } - -// newHolder reads the paused flag the workspace keeps. -func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder { - h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun} - if _, err := os.Stat(pausedFile(workspace)); err == nil { - h.paused = true - } - return h -} - -// Paused is asked by the taking loop before every fetch. -func (h *holder) Paused() bool { - h.mu.Lock() - defer h.mu.Unlock() - return h.paused -} - -// setPaused records the flag in the workspace first and then in memory, so what this holder says -// it is and what it would be after a restart never differ. -func (h *holder) setPaused(paused bool) error { - path := pausedFile(h.workspace) - if paused { - if err := os.MkdirAll(h.workspace, 0o755); err != nil { - return err - } - if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil { - return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err) - } - } else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("cannot remove the paused flag %s: %w", path, err) - } - h.mu.Lock() - h.paused = paused - h.mu.Unlock() - h.announce() - return nil -} - -// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the -// controller reading an older state is a plan read as late rather than a build lost. -func (h *holder) announce() { - if h.say == nil { - return - } - if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil { - fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err) - } -} - -// stateWhilePaused says this machine's state at start, and again every while it stays paused, so -// a pause outlives the events stream's retention. -func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) { - h.announce() - tick := time.NewTicker(every) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - if h.Paused() { - h.announce() - } - } - } -} - -// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while -// after. Every holder sweeps the one store; what one let go of, another finds gone. -func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) { - sweep := func() { - if h.scratch == nil { - return - } - n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now()) - switch { - case err != nil: - fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err) - case n > 0: - fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n) - } - } - sweep() - tick := time.NewTicker(every) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - sweep() - } - } -} - -// begin records the build this machine took, with the cancel that ends it. -func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running { - r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})} - h.mu.Lock() - h.running = r - h.mu.Unlock() - return r -} - -// end clears it, and lets a kill waiting on it know it is over. -func (h *holder) end(r *running) { - h.mu.Lock() - if h.running == r { - h.running = nil - } - h.mu.Unlock() - close(r.done) -} - -// stepped records the step a build is at, for `current`. -func (h *holder) stepped(r *running, step string) { - h.mu.Lock() - r.step = step - h.mu.Unlock() -} - -// currentBuild is what `current` answers. -type currentBuild struct { - On string `json:"on"` - Paused bool `json:"paused"` - Running *struct { - ID string `json:"id"` - Repository string `json:"repository"` - Path string `json:"path,omitempty"` - Ref string `json:"ref,omitempty"` - Step string `json:"step,omitempty"` - Started string `json:"started"` - Elapsed string `json:"elapsed"` - } `json:"running,omitempty"` - Said string `json:"said"` -} - -func (h *holder) current() currentBuild { - h.mu.Lock() - defer h.mu.Unlock() - out := currentBuild{On: h.on, Paused: h.paused} - taking := "taking builds" - if h.paused { - taking = "paused, taking no new build" - } - if h.running == nil { - out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking) - return out - } - r := h.running - elapsed := time.Since(r.started).Round(time.Second) - out.Running = &struct { - ID string `json:"id"` - Repository string `json:"repository"` - Path string `json:"path,omitempty"` - Ref string `json:"ref,omitempty"` - Step string `json:"step,omitempty"` - Started string `json:"started"` - Elapsed string `json:"elapsed"` - }{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step, - r.started.UTC().Format(time.RFC3339), elapsed.String()} - out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s", - h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking) - return out -} - -// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between -// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer -// (killAnswer in the controller's queue.go, 75s). -var ( - killRemoves = 15 * time.Second - killWaits = 20 * time.Second -) - -// kill ends the build with this id, if it is the one running here and still working: its context -// cancelled — which kills each command's process group — and the containers it started removed by -// their label, once at once and again after the build has ended, so one created while it was being -// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the -// ask so it is not redelivered; the answer says whether that happened. -func (h *holder) kill(id string) (string, error) { - h.mu.Lock() - r := h.running - if r == nil || r.request.ID != id { - doing := "nothing" - if r != nil { - doing = r.request.ID - } - h.mu.Unlock() - return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing) - } - if r.returned { - h.mu.Unlock() - return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on) - } - r.killed = true - cancel, done := r.cancel, r.done - h.mu.Unlock() - - cancel() - removed, removeErr := h.removeContainers(id) - ended := false - select { - case <-done: - ended = true - case <-time.After(killWaits): - } - again, againErr := h.removeContainers(id) - removed += again - if removeErr == nil { - removeErr = againErr - } - containers := fmt.Sprintf("%d container(s) it started removed", removed) - if removeErr != nil { - containers = "its containers could not all be listed or removed: " + removeErr.Error() - } - if !ended { - return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+ - "its outcome is in", id, h.on, containers), nil - } - h.mu.Lock() - outcome, announced := r.outcome, r.announced - h.mu.Unlock() - if !announced { - return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+ - "announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on, - containers), nil - } - return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+ - "settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil -} - -// removeContainers is one pass of removing what the build left, bounded. -func (h *holder) removeContainers(id string) (int, error) { - cleanup, stop := context.WithTimeout(context.Background(), killRemoves) - defer stop() - return builder.RemoveContainersOf(cleanup, h.remove, id) -} - -// removeLeftBehind removes, once, what earlier holders on this machine left — called before anything is -// taken, so none of it is this holder's — and says what it did. A runtime that cannot be asked is said -// and the holder takes work all the same: what is left is waste, not a reason to stop building. -func (h *holder) removeLeftBehind() int { - cleanup, stop := context.WithTimeout(context.Background(), killRemoves) - defer stop() - n, err := builder.RemoveLeftBehind(cleanup, h.remove) - switch { - case err != nil: - fmt.Fprintf(os.Stderr, "could not look for containers earlier builds left on %s: %v\n", h.on, err) - case n > 0: - fmt.Fprintf(os.Stderr, "removed %d container(s) earlier builds left on %s\n", n, h.on) - } - return n -} - -// returned records that the build's work ended, and says whether a kill came first — only then is -// the build killed; an error it ended with on its own is its own outcome. -func (h *holder) returned(r *running) bool { - h.mu.Lock() - defer h.mu.Unlock() - r.returned = true - return r.killed -} - -// said records the outcome announced, and whether it went out, for a kill to answer with. -func (h *holder) said(r *running, outcome string, announced bool) { - h.mu.Lock() - r.outcome, r.announced = outcome, announced - h.mu.Unlock() -} - -// handlers are the seat's verbs, as this machine answers them. -func (h *holder) handlers() map[string]link.ToolHandler { - return map[string]link.ToolHandler{ - "current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil }, - "kill": func(_ context.Context, raw json.RawMessage) (any, error) { - var args struct { - ID string `json:"id"` - } - if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" { - return nil, errors.New("kill needs the build's id") - } - said, err := h.kill(strings.TrimSpace(args.ID)) - if err != nil { - return nil, err - } - return map[string]any{"said": said}, nil - }, - "pause": func(context.Context, json.RawMessage) (any, error) { - if err := h.setPaused(true); err != nil { - return nil, err - } - said := h.on + " is paused: it takes no new build until resumed" - if c := h.current(); c.Running != nil { - said += "; " + c.Running.ID + " runs on and finishes" - } - return map[string]any{"said": said, "paused": true}, nil - }, - "resume": func(context.Context, json.RawMessage) (any, error) { - if err := h.setPaused(false); err != nil { - return nil, err - } - return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil - }, - } -} - -func orNothing(s string) string { - if s == "" { - return "its start" - } - return s -} - -// plainRun runs a command outside any build: no line reaches a build's log, because the build whose -// log it would be is the one being ended. -func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) { - cmd := exec.CommandContext(ctx, name, args...) - cmd.Dir = dir - out, err := cmd.CombinedOutput() - if err != nil { - return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) - } - return string(out), nil -} - -// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this -// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat, -// the module answering, the seat, scope node, the machine, its description and argument schema — so -// the console finds `/node-build-agent.kill` by searching, as it finds any seat's verb. -// -// One service per machine, named for the seat and identified by the machine, so the answer is this -// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no -// store, and what it serves is what this binary was built to serve. -func announcement(seat, module, node string) micro.Info { - s, _ := catalogue.SeatNamed(seat) - var endpoints []micro.EndpointInfo - for _, v := range s.Serves { - schema, _ := json.Marshal(v.Input) - endpoints = append(endpoints, micro.EndpointInfo{ - Name: seat + "__" + v.Name, - Subject: link.NodeSeatToolSubject(seat, v.Name, node), - // The queue group the verbs are served in, as every runtime announces its own. - QueueGroup: "seat." + seat, - Metadata: map[string]string{ - "kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node", - "node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema), - }, - }) - } - return micro.Info{ - ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0", - Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}}, - Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)", - Endpoints: endpoints, - } -} - -// moduleOf is the module a credential was issued for: its user is `.`. -func moduleOf(user string) string { - if _, module, ok := strings.Cut(user, "."); ok && module != "" { - return module - } - return "build-agent" -} diff --git a/cmd/mesh-builder/holder_test.go b/cmd/mesh-builder/holder_test.go deleted file mode 100644 index c89c6b3d..00000000 --- a/cmd/mesh-builder/holder_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/link" -) - -// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR -// 0219), and what it says about itself follows. -func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) { - workspace := t.TempDir() - var said []link.HolderState - h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error { - said = append(said, s) - return nil - }) - if h.Paused() { - t.Fatal("a new holder starts paused") - } - if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil { - t.Fatal(err) - } - if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" { - t.Fatalf("paused: %v, said %+v", h.Paused(), said) - } - again := newHolder("ace", link.TheBuildMachine, workspace, nil) - if !again.Paused() { - t.Fatal("restarted, the holder forgot it was paused") - } - if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil { - t.Fatal(err) - } - if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() { - t.Fatal("resumed, the holder came back paused") - } -} - -// kill ends the build with that id — its context, which ends its commands — removes what it left by -// label, and refuses an id it is not building. -func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) { - h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil) - var removed []string - h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) { - removed = append(removed, name+" "+strings.Join(args, " ")) - if args[0] == "ps" { - return "c1\n", nil - } - return "", nil - } - kill := h.handlers()["kill"] - if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil { - t.Fatal("killed a build while none ran") - } - - building, cancel := context.WithCancel(context.Background()) - r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel) - h.stepped(r, "image") - if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" { - t.Fatalf("current says %+v", c) - } - if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil || - !strings.Contains(err.Error(), "build-1") { - t.Fatalf("killed another id: %v", err) - } - // The build's own goroutine: it ends when its context does, as a build's commands do, and - // announces what came of it. - go func() { - <-building.Done() - if h.returned(r) { - h.said(r, link.KilledByHand, true) - } - h.end(r) - }() - answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)) - if err != nil { - t.Fatal(err) - } - if building.Err() == nil { - t.Fatal("the build's context was not cancelled") - } - if !r.killed { - t.Error("the build is not marked killed, so it would be announced as an ordinary failure") - } - said := answer.(map[string]any)["said"].(string) - if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) { - t.Errorf("kill said %q", said) - } - // Removed at the kill and again once the build had ended: a container made in between is caught. - if len(removed) != 6 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[3], "label=mesh.build=build-1") { - t.Errorf("removed %v", removed) - } - if c := h.current(); c.Running != nil { - t.Errorf("after the kill current says %+v", c) - } -} - -// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more. -func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) { - info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace") - if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 { - t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints)) - } - kill := info.Endpoints[1] - md := kill.Metadata - if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" || - md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" || - !strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) { - t.Fatalf("kill is announced as %+v", kill) - } - body, err := json.Marshal(info) - if err != nil || len(body) > 8*1024 { - t.Fatalf("the answer is %d bytes (%v)", len(body), err) - } -} - -// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its -// own error is its outcome. One whose outcome could not be announced is not said to be settled. -func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) { - h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil) - h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil } - _, cancel := context.WithCancel(context.Background()) - r := h.begin(link.BuildRequest{ID: "build-1"}, cancel) - if killed := h.returned(r); killed { - t.Fatal("a build nobody killed reads as killed") - } - if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") { - t.Fatalf("killed a build that had ended: %v", err) - } - h.end(r) - - building, cancel := context.WithCancel(context.Background()) - r = h.begin(link.BuildRequest{ID: "build-2"}, cancel) - go func() { - <-building.Done() - if h.returned(r) { - h.said(r, link.KilledByHand, false) - } - h.end(r) - }() - said, err := h.kill("build-2") - if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") { - t.Fatalf("kill said %q (%v)", said, err) - } -} - -// **Issue 306**: a holder starting removes the containers earlier builds left on its machine, through -// the runner a kill uses, and a runtime that cannot be asked does not stop it. -func TestAHolderStartingRemovesWhatEarlierBuildsLeftHere(t *testing.T) { - h := newHolder("novox", link.TheBuildMachine, t.TempDir(), nil) - var removed []string - h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) { - removed = append(removed, name+" "+strings.Join(args, " ")) - if args[0] == "ps" { - return "c1 build-1\nc2 build-1\nc3 check-here-2\n", nil - } - return "", nil - } - if n := h.removeLeftBehind(); n != 2 || len(removed) != 3 || removed[2] != "docker rm -f -v c1 c2" { - t.Fatalf("removed %d: %v", n, removed) - } - h.remove = func(context.Context, string, string, ...string) (string, error) { - return "", errors.New("no runtime") - } - if n := h.removeLeftBehind(); n != 0 { - t.Errorf("a runtime that cannot be asked removed %d", n) - } -} diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go deleted file mode 100644 index 2945e462..00000000 --- a/cmd/mesh-builder/main.go +++ /dev/null @@ -1,946 +0,0 @@ -// mesh-builder — the thing a build machine runs. -// -// It takes work from the mesh, turns a repository into artifacts, publishes them, and says what -// came out. It is **not** the control plane and it is **not** the host: -// -// - the control plane decides and never touches a machine. Building runs commands on one, and -// what the control plane may send a machine is bounded by the declaration language -// (novox/hq ADR 0005). "Run this build" is not in it, and widening the language so it could -// be would make the control plane able to run anything anywhere. -// - the host applies declarations and holds no opinion about what they contain. A host that -// also built things would need a container runtime and git, on every machine, to do something -// almost none of them will ever do. -// -// So it is a module: a program a machine runs because the mesh told it to, holding its own broker -// credential and nothing else. Compromise of a build machine is compromise of a build machine. -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log" - "net/url" - "os" - "os/signal" - "sort" - "strings" - "sync" - "sync/atomic" - "syscall" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/builder" - "github.com/novox/mesh-controller/internal/link" -) - -// version is set at build time. -var version = "development" - -func main() { - if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "mesh-builder: %v\n", err) - os.Exit(1) - } -} - -const usage = `mesh-builder — builds modules for the mesh - -It consumes build requests and answers with what it made. Nothing is listened on and nothing -is dialled except the broker. - - MESH_BROKER_FILE a file the mesh sealed to this machine holding the same - MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said - MESH_BINDING a file the mesh wrote saying where the artifact store is - MESH_PACKAGE_BINDING a file the mesh wrote saying where the package registry is - MESH_NPM_TOKEN_FILE a file the mesh sealed holding the token for it - MESH_GENESIS_PACKAGE_REGISTRY the document the installer sealed at genesis, read while no - binding is present (novox/hq ADR 0336) - MESH_NPM_REGISTRY a package registry URL, when the mesh has not said (a person, the bootstrap) - MESH_NPM_TOKEN the token for it, likewise - MESH_NPM_SCOPE the scope it answers for (default: @novox) - MESH_WORKSPACE where to clone and build (default: a temporary directory) - MESH_BUILD_CGROUP_PARENT the slice every container a build or check starts joins, which carries the - machine's limits on builds (novox/hq issue 1798; default: the runtime's own) - -It also builds one module and stops, which is how a mesh is raised — before there is a -broker to take work from or a registry to publish into: - - mesh-builder build [--path P] [--ref COMMIT] [--registry HOST:PORT] - -Without --registry the artifacts stay in this machine's container runtime, named by the -digest of their own configuration. The result is printed as JSON. - - mesh-builder ready - -exits 0 once the agent running in this container is bound to its build work and pulls it, and -1 before: its container's health check. - - mesh-builder busy - -exits 0 while the agent running in this container has a build in hand, and 1 when it has none; -from the first ask the agent takes no new build: its container's busy command, asked before the -container is replaced (novox/hq ADR 0335). -` - -func run() error { - if len(os.Args) > 1 { - switch os.Args[1] { - case "version": - fmt.Println(version) - return nil - case "build": - return buildOnce(context.Background(), os.Args[2:]) - case "ready": - // The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and - // once it has been drained longer than its node-engine would wait to replace it (ADR 0335). - if err := ready(); err != nil { - return err - } - return drainedTooLong(time.Now()) - case "busy": - // The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once - // nothing is in hand. - said, err := busy() - if err != nil { - return err - } - fmt.Println(said) - return nil - case "help", "-h", "--help": - fmt.Print(usage) - return nil - default: - // **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked - // it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound. - fmt.Fprint(os.Stderr, usage) - return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1]) - } - } - - if err := notReady(); err != nil { - return err - } - // **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds - // differently never takes a build another one made. - builder.Version = builder.SelfVersion(version) - if err := undrained(); err != nil { - return err - } - credential, err := brokerFrom() - if err != nil { - return err - } - registry, err := whereToPublish() - if err != nil { - return err - } - // A build and a check run below the machine's own work (novox/hq issue 1798): the agent itself at the - // lowest priority, which its git and go children inherit, and every container in the build slice. - _ = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 19) - builder.CgroupParent = strings.TrimSpace(os.Getenv("MESH_BUILD_CGROUP_PARENT")) - workspace := os.Getenv("MESH_WORKSPACE") - if workspace == "" { - workspace = os.TempDir() + "/mesh-builder" - } - // **The mesh's name for this machine, not the container's.** A build is reported to the rest - // of the mesh, and a report whose origin reads `104cb10e105b` names something no other module - // can look up. The mesh already knows the answer and has a way to say it — `${machine:name}` - // in the environment file this module is handed — so the hostname is only what is left when - // nobody said. - on := os.Getenv("MESH_NODE") - if on == "" { - hostname, err := os.Hostname() - if err != nil { - return fmt.Errorf("this build machine has no name: nothing said MESH_NODE and the host would not say either: %w", err) - } - on = hostname - } - - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - - js, seat, err := dialFor(credential) - if err != nil { - return err - } - defer js.Close() - - // **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The - // paused flag is read from the workspace before anything is taken, so a holder restarted while - // paused takes nothing. - h := newHolder(on, seat, workspace, func(state link.HolderState) error { - body, err := json.Marshal(state) - if err != nil { - return err - } - _, err = js.Context().Publish(link.BuildPausedOf(seat, on), body) - return err - }) - if h.Paused() { - fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace)) - } - // **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder - // recreated mid-check left the check's store and bus running, and the ask went to another machine. - h.removeLeftBehind() - // Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build - // in hand and then says idle, and its container is replaced between builds. - machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{ - Paused: func() bool { return h.Paused() || drained() || markFaulty(time.Now()) || noRoom(workspace) }, - Taking: func() { taking(seat) }, - Claim: claim, - }) - defer machine.Close() - - // The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired - // one serves none, and a subscription its holder has no grant for would be refused for ever. - if seat == link.TheBuildMachine { - stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(), - log.New(os.Stderr, "", 0)) - if err != nil { - return err - } - defer stopServing() - // And says so, for the console to find (novox/hq ADR 0197). - stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce( - announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0)) - if err != nil { - return err - } - defer stopAnnouncing() - go h.stateWhilePaused(ctx, time.Hour) - } - - fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry) - publisher := builder.Registry{Address: registry, Run: builder.Command} - // Check builds are let go of after a week (novox/hq ADR 0350 §5). - h.scratch = publisher - go h.expireChecks(ctx, time.Hour, time.Now) - - return machine.Take(ctx, func(ctx context.Context, work link.Build) { - defer released() - holding(work.Request().Repository + " (" + work.Request().ID + ")") - answer(ctx, publisher, on, workspace, work, h) - }) -} - -// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it -// holds. -// -// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build -// machine told about both would take work from one and answer on the other, and every log line would -// say it was fine. -func dialFor(credential Credential) (*broker.JetStream, string, error) { - // **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5). - // A credential for the mesh's bus carries user, password and fingerprint beside the address, - // and that is enough to dial it, pinned. - if !credential.onTheNewBus() { - return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL) - } - // Named for what it is, not the controller whose code dials it (novox/hq issue 327). - host, _ := os.Hostname() - js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint, nats.Name("build agent on "+host)) - if err != nil { - return nil, "", err - } - // **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the - // handover): the mesh issues a build machine's credential naming the seat its module claims, - // and one binary serves the old role as `builder` and the new as `build-agent` from that alone. - seat := link.BuildSeatClaimed(credential.seatsClaimed()) - fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat) - return js, seat, nil -} - -// answer does one build and says what happened, whichever way it went. -func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) { - request := work.Request() - - // **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends - // this build's commands and nothing else; the machine's own context ending — a SIGTERM — still - // reaches it through the parent, and that keeps today's meaning below. - building, cancel := context.WithCancel(ctx) - defer cancel() - var mine *running - if h != nil { - mine = h.begin(request, cancel) - defer h.end(mine) - } - - // **First thing, and to stdout.** A build request that arrives and produces no visible line until - // it either finishes or fails is indistinguishable from one that never arrived — which cost a long - // diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that - // the handler said nothing until the end. - fmt.Fprintf(os.Stderr, "a build request arrived for %s (%s)\n", request.Repository, request.ID) - - // **Everything a build says goes two ways**: to stderr, as always, and onto the bus as the - // role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody - // watching, reads the same lines this container's log holds, live, and after the fact from the - // stream. Said first, before anything runs, so a build that hangs is one that visibly started. - // One line at a time: a check's builds for its merge say their lines while its tests say theirs (novox/hq - // issue 716). - var saying sync.Mutex - say := func(step, message string) { - saying.Lock() - defer saying.Unlock() - fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) - work.Say(step, message) - if mine != nil && step != "run" && step != "output" { - h.stepped(mine, step) - } - } - builder.Said = say - defer func() { builder.Said = nil }() - if err := work.Began(ctx); err != nil { - fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err) - } - // **A running check watches its pull request's key** (novox/hq ADR 0406 §4, issue 974): when the head or the trunk - // commit in it moves, the check is for a state nobody wants any more, and ends itself — nothing is asked of the forge - // (issue 726). A group's check, a try and a build watch nothing. - var superseded atomic.Bool - if w, ok := work.(interface { - WatchPull(context.Context, func(link.PullKey)) (func(), error) - }); ok && request.Check != nil { - stopWatching, err := w.WatchPull(building, func(k link.PullKey) { - superseded.Store(true) - say("superseded", fmt.Sprintf("the pull request's key moved to %.12s on %.12s while this checked %.12s on %.12s: stopped", - k.Head, k.Trunk, request.Ref, request.Check.Trunk)) - cancel() - }) - if err != nil { - fmt.Fprintf(os.Stderr, "the key of %s/%s#%d could not be watched, so its check runs on if the key moves: %v\n", - request.Check.Owner, request.Check.Repo, request.Check.Number, err) - } else { - defer stopWatching() - } - } - - result := link.BuildResult{ - ID: request.ID, Repository: request.Repository, Path: request.Path, - Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun, - } - what := "building " + request.Repository - if request.Path != "" { - what += " at " + request.Path - } - if request.Ref != "" { - what += " on " + request.Ref - } - say("build", what) - - npmrc, err := packagesFrom(on) - if err == nil { - say("packages", saysRegistry(npmrc)) - } - registryHere, isRegistry := publisher.(builder.Registry) - var built builder.Result - if request.Check != nil { - // **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built, - // published or registered; the verdict is the outcome. - result.Checked = request.Check - registry := "" - if r, ok := publisher.(builder.Registry); ok { - registry = r.Address - } - spec := checkSpecOf(request) - // The judge cache the build agent declares as its own data, never in the workspace: only judge builds and - // mesh-lab's replays mount it, never a check's containers, so no pull request's script can write what a judge - // is built with or kept as, or what the replays run (novox/hq ADR 0358 §6, issue 709). - spec.JudgeCache = judgeCache() - // What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the - // scratch namespace. **Never with the package registry's credential**: this is code nobody has approved - // yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages - // at build time has no reuse key anyway, so nothing is lost by refusing it here. - if isRegistry && len(spec.Builds) > 0 { - spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held, - Running: request.Check.Running, Seats: request.Seats, Named: namedOf(request.Check.Named)} - } - var v builder.CheckVerdict - v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say) - if err == nil { - result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report, - Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo), - Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built), - Validator: v.Validator, Siblings: siblingsOut(v.Siblings), SiblingsUnknown: v.SiblingsUnknown, JudgeFrom: v.JudgeFrom} - // What each of a group's heads was judged as (novox/hq ADR 0323 §6). - for _, m := range v.Members { - result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, - Commit: m.Head, Trunk: m.Trunk}) - } - // A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686). - if request.Check.Try { - if line, err := json.Marshal(triedOf(v)); err == nil { - say(link.StepTried, string(line)) - } - } - } - } else if err == nil { - // The package-registry credential is a build input, so it is resolved before the clone: a - // build that could not have resolved its dependencies is refused in front of the reason, not - // after a clone that then fails at npm ci. - // Every container it starts is labelled with its id, so a kill finds what outlived the - // docker client (ADR 0219). - if request.Reuse != nil && isRegistry && !request.DryRun { - // **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2). - reuse := builder.Reuse{Scratch: registryHere} - for _, p := range request.Reuse.Pulls { - reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number}) - } - // Only what a verdict names is taken (ADR 0350 §2). - reuse.Named = namedOf(request.Reuse.Builds) - built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse, - request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, - forgeFrom(), say, request.Seats) - } else { - built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher, - request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, - forgeFrom(), say, request.Seats) - } - } - // What it copied into the store, whatever became of the build (novox/hq ADR 0257). - result.Mirrored = built.Mirrored - // Only a build the kill ended: the kill came before its work did. One that finished — built, or - // failed on its own — in the moment the kill arrived says what it did, and the kill is refused. - killed := false - if mine != nil { - killed = h.returned(mine) && err != nil - } - if superseded.Load() && err != nil { - // Ended by its own key moving on (ADR 0406 §4): said as the check's failure, as a kill is. - result.Failed = link.SupersededByKey - say("failed", link.SupersededByKey) - } else if killed { - // **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the - // error it ended with is the kill's consequence, not a fault of the source. - result.Failed = link.KilledByHand - say("failed", link.KilledByHand) - } else if err != nil { - // A failure is a result. A build that fails and says nothing is indistinguishable from a - // builder that is not running, and those want completely different responses. - result.Failed = err.Error() - say("failed", err.Error()) - } else if request.Check != nil { - say("checked", result.Check.Verdict+": "+result.Check.Summary) - } else { - manifest, marshalErr := json.Marshal(built.Manifest) - if marshalErr != nil { - result.Failed = marshalErr.Error() - } else { - result.Commit = built.Commit - result.Manifest = manifest - for _, made := range built.Built { - result.Made = append(result.Made, link.MadeArtifact{ - Name: made.Name, Kind: made.Kind, Reference: made.Reference, - }) - } - result.Against = built.Against - result.SourceFingerprint = built.Source - result.SourceInputs = built.SourceInputs - // What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4). - result.ReuseKey = built.ReuseKey - if built.Reused != nil { - f := built.Reused.From - result.Reused = built.Reused.Words() - result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo, - Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key} - } else if request.Reuse != nil { - result.BuiltWhy = built.Why - if result.BuiltWhy == "" { - result.BuiltWhy = "it has no artifact to build" - } - } - result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches - for _, r := range built.Read { - result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref}) - } - for _, s := range built.Sources { - result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) - } - // The list of predefined keys the manifest was judged against, for the controller to publish - // (novox/hq ADR 0348 §7). - if p := built.Predefined; p != nil { - result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation} - } - say("built", built.Manifest.Module+" from "+short(built.Commit)) - } - } - - // A killed build is announced on a context of its own: the build's was the one cancelled, and the - // outcome must go out and the ask be settled — acknowledged, never redelivered to another machine - // to be built again. A machine being stopped is the other case and keeps its meaning: the - // parent's context is gone, nothing is announced or settled, and the ask is redelivered. - announcing := ctx - if killed { - fresh, stop := context.WithTimeout(context.Background(), 30*time.Second) - defer stop() - announcing = fresh - } - announceErr := work.Announce(announcing, result) - if mine != nil { - h.said(mine, result.Failed, announceErr == nil) - } - if err := announceErr; err != nil { - // Said, not fatal: the build happened. A build reported as failed because announcing it - // failed is a lie about work that was done — and the request stays unsettled below only if - // nothing was said at all, so another machine can try. - fmt.Fprintf(os.Stderr, "cannot say what came of a build: %v\n", err) - return - } - - // Settled only once the outcome is away, so a machine that dies before answering leaves the work - // for another rather than losing it. - if err := work.Done(); err != nil { - fmt.Fprintf(os.Stderr, "the outcome is away and the request could not be settled: %v\n", err) - } -} - -// checkSpecOf is a check request as the builder runs it. -func checkSpecOf(request link.BuildRequest) builder.CheckSpec { - c := request.Check - spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref, - Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{}, - Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base, - Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held), - Tests: c.Tests, StoreOnly: c.StoreOnly, SiblingsUnknown: c.SiblingsUnknown} - for _, h := range c.HeldConfig { - spec.Held = append(spec.Held, builder.HeldValue{Pull: h.Pull, Head: h.Head, Node: h.Node, Module: h.Module, Key: h.Key, - Value: h.Value, Approved: h.Approved, By: h.By, At: h.At}) - } - for _, s := range c.Siblings { - spec.Siblings = append(spec.Siblings, builder.Sibling{Repository: s.Repository, Number: s.Number, Head: s.Head, - Base: s.Base, Dir: s.Dir}) - } - for dir, b := range c.Beside { - spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref} - } - for _, b := range c.Builds { - spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path, - Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip}) - } - for _, m := range c.Members { - spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository, - Ref: m.Ref, Paths: m.Paths, Base: m.Base}) - } - return spec -} - -// siblingsOut is the siblings a check read as the outcome carries them (novox/hq ADR 0382 §4). -func siblingsOut(read []builder.JudgedSibling) []link.CheckSibling { - var out []link.CheckSibling - for _, s := range read { - out = append(out, link.CheckSibling{Repository: s.Repository, Number: s.Number, Head: s.Head, Base: s.Base, - BaseCommit: s.BaseCommit}) - } - return out -} - -// judgeCache is the judge cache the build agent's manifest declares (MESH_JUDGE_CACHE, novox/hq ADR 0358 §6): a -// directory mounted at the same path on the host, so a judge build's mounts name what the agent sees. Empty when the -// agent declares none, and every judge is then built cold under its check's root. -func judgeCache() string { return os.Getenv("MESH_JUDGE_CACHE") } - -// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict, -// or the check's where that did not run — a head that does not merge — and the lines naming what failed. -func triedOf(v builder.CheckVerdict) link.Tried { - t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(), - Head: v.Head, Trunk: v.Trunk} - if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") { - t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary - } - if v.Repo != nil && v.Repo.Failed != "" { - t.Failed = strings.Split(v.Repo.Failed, "\n") - } - if t.Verdict == "" { - t.Verdict = "error" - } - return t -} - -// namedOf are the builds the seat's verdicts name, as the builder believes a scratch record by them (novox/hq ADR -// 0350 §2, issue 703): each module's key with every artifact's digest. -func namedOf(builds []link.CheckedBuild) []builder.NamedBuild { - var out []builder.NamedBuild - for _, b := range builds { - named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo, - Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key, - Digests: map[string]string{}} - for _, m := range b.Built.Made { - named.Digests[m.Name] = builder.DigestOf(m.Reference) - } - out = append(out, named) - } - return out -} - -// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1). -func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt { - var out []link.CheckBuilt - for _, b := range built { - c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String(), - Kept: b.Kept, Same: b.Same, SameAs: b.SameAs} - for _, m := range b.Made { - c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference}) - } - out = append(out, c) - } - return out -} - -// layerOf is one layer of a check as the outcome carries it. -func layerOf(l *builder.Layer) *link.CheckLayer { - if l == nil { - return nil - } - return &link.CheckLayer{Verdict: l.Verdict, Summary: l.Summary, Modules: l.Modules} -} - -// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all -// (novox/hq ADR 0076, issue 053) — on this machine. -// -// **The binding wins, then the genesis document** (novox/hq ADR 0336 §3, §4). The build agent's -// npm-package-registry requirement is optional: on genesis's node, before the gitea module is assigned, -// nothing provides it, and the installer sealed the forge it raised to this agent as one document -// instead. Once the requirement binds, the binding names the endpoint the way the artifact store's -// does and a sealed token file the credential, and the genesis document is left unread. A binding -// file named and absent is a requirement not bound here, never a fault: the environment names the -// path whether or not the mesh wrote it. -// -// The environment variables remain for a builder run by a person. With none of the three the result -// is disabled and says this machine, so a build that needs the registry is refused naming it (§6), -// and a build that needs none builds anyway. -func packagesFrom(machine string) (builder.Npmrc, error) { - scope := strings.TrimSpace(os.Getenv("MESH_NPM_SCOPE")) - if scope == "" { - scope = "@novox" - } - none := builder.Npmrc{Machine: machine} - - if path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")); path != "" { - raw, err := os.ReadFile(path) - switch { - case errors.Is(err, os.ErrNotExist): - // Not bound on this machine: the genesis document, or nothing. - case err != nil: - return none, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err) - default: - return fromBinding(path, raw, scope, machine) - } - } - if path := strings.TrimSpace(os.Getenv("MESH_GENESIS_PACKAGE_REGISTRY")); path != "" { - raw, err := os.ReadFile(path) - switch { - case errors.Is(err, os.ErrNotExist): - // Not this mesh's genesis node, or the installer has not sealed it yet. - case err != nil: - return none, fmt.Errorf("cannot read the genesis document for the package registry: %w", err) - default: - return fromGenesis(path, raw, scope, machine) - } - } - - // A builder run by a person. - registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY")) - secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN")) - username := strings.TrimSpace(os.Getenv("MESH_NPM_USER")) - if registry == "" && secret == "" { - return none, nil - } - n := builder.Npmrc{Scope: scope, Registry: registry, From: builder.FromEnvironment, Machine: machine} - if username != "" { - n.Username, n.Password = username, secret - } else { - n.Token = secret - } - return n, nil -} - -// fromBinding composes the registry from the npm-package-registry binding the mesh wrote and the -// credential it sealed beside it. The mesh authenticates the ordinary way — a generated password the -// provider only applies (novox/hq ADR 0048) — so with a username the secret is a password (basic -// auth); without one it is a bearer token a provider minted. -func fromBinding(path string, raw []byte, scope, machine string) (builder.Npmrc, error) { - none := builder.Npmrc{Machine: machine} - var told struct { - From string `json:"from"` - At string `json:"at"` - As string `json:"as"` - Serves map[string]any `json:"serves"` - } - if err := json.Unmarshal(raw, &told); err != nil { - return none, fmt.Errorf("%s is not a binding: %w", path, err) - } - if told.At == "" { - return none, fmt.Errorf( - "%s says the package registry is on %q and gives no address for it", path, told.From) - } - // Composed from what the provider serves, so nothing here knows gitea's URL shape from - // another registry's: it states its port, the path its registry answers on, and the scheme. - scheme := "https" - if s, ok := told.Serves["scheme"]; ok { - scheme = fmt.Sprintf("%v", s) - } - port, ok := told.Serves["port"] - if !ok { - return none, fmt.Errorf("%s says nothing about which port the package registry answers on", path) - } - npmPath, ok := told.Serves["npm-path"] - if !ok { - return none, fmt.Errorf("%s says nothing about the path the package registry answers on", path) - } - secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN")) - if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" { - raw, err := os.ReadFile(file) - if err != nil { - return none, fmt.Errorf("cannot read this builder's package-registry credential: %w", err) - } - secret = strings.TrimSpace(string(raw)) - } - n := builder.Npmrc{Scope: scope, Registry: fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath), - From: builder.FromBinding, Machine: machine} - if told.As != "" { - n.Username, n.Password = told.As, secret - } else { - n.Token = secret - } - return n, nil -} - -// GenesisRegistry is the genesis document (novox/hq ADR 0336 §3): what the installer sealed to the -// build agent on its node at genesis's packages step, as the `genesis-package-registry` own-secret — -// the forge it raised, the port this node gave the package registry, its npm path, and the account it -// made with that account's password. mesh-host writes it; the field names are the two repositories' -// agreement. -type GenesisRegistry struct { - Scheme string `json:"scheme"` - At string `json:"at"` - Port int `json:"port"` - NpmPath string `json:"npm-path"` - As string `json:"as"` - Password string `json:"password"` -} - -// fromGenesis composes the registry from the genesis document. -func fromGenesis(path string, raw []byte, scope, machine string) (builder.Npmrc, error) { - none := builder.Npmrc{Machine: machine} - var g GenesisRegistry - if err := json.Unmarshal(raw, &g); err != nil { - return none, fmt.Errorf("%s is not the genesis document for the package registry: %w", path, err) - } - var missing []string - for name, value := range map[string]string{"at": g.At, "npm-path": g.NpmPath, "as": g.As, "password": g.Password} { - if strings.TrimSpace(value) == "" { - missing = append(missing, name) - } - } - if g.Port <= 0 { - missing = append(missing, "port") - } - if len(missing) > 0 { - sort.Strings(missing) - return none, fmt.Errorf("%s, the genesis document for the package registry, says nothing of %s", - path, strings.Join(missing, ", ")) - } - scheme := g.Scheme - if scheme == "" { - scheme = "http" - } - return builder.Npmrc{Scope: scope, Registry: fmt.Sprintf("%s://%s:%d%s", scheme, g.At, g.Port, g.NpmPath), - Username: g.As, Password: g.Password, From: builder.FromGenesis, Machine: machine}, nil -} - -// saysRegistry is the line each build's log carries about the package registry (novox/hq ADR 0336 -// §4): which of the binding and the genesis document it resolved from, or that it has neither here. -func saysRegistry(n builder.Npmrc) string { - if n.Enabled() { - return "the package registry is resolved from " + n.From - } - machine := n.Machine - if machine == "" { - machine = "this machine" - } - return fmt.Sprintf("no package registry: %s is unbound on %s and no genesis document is here; "+ - "a build needing it is refused", builder.RegistryRequirement, machine) -} - -// forgeFrom is the git credential this builder may offer a clone, composed from the same binding -// and sealed secret its package-registry half already reads: the forge that answers npm is the -// forge that hosts the repositories, and its provisioner applies one password to one user for -// both. Anything missing means no credential, and every clone stays anonymous — which is all a -// mesh of public repositories ever needs. -// -// The URL names the binding's own address — the machine the mesh says the forge is on — so a -// private repository is registered and built by that address, and a clone of anything else is -// never shown this credential (git's credential store matches the whole origin). -func forgeFrom() builder.GitCredential { - path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")) - if path == "" { - return builder.GitCredential{} - } - raw, err := os.ReadFile(path) - if err != nil { - return builder.GitCredential{} - } - var told struct { - At string `json:"at"` - As string `json:"as"` - Serves map[string]any `json:"serves"` - } - if err := json.Unmarshal(raw, &told); err != nil || told.At == "" || told.As == "" { - return builder.GitCredential{} - } - secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN")) - if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" { - if raw, err := os.ReadFile(file); err == nil { - secret = strings.TrimSpace(string(raw)) - } - } - if secret == "" { - return builder.GitCredential{} - } - scheme := "https" - if s, ok := told.Serves["scheme"]; ok { - scheme = fmt.Sprintf("%v", s) - } - host := told.At - if port, ok := told.Serves["port"]; ok { - host = fmt.Sprintf("%s:%v", told.At, port) - } - made := url.URL{Scheme: scheme, User: url.UserPassword(told.As, secret), Host: host} - return builder.GitCredential{URL: made.String()} -} - -func short(commit string) string { - if len(commit) > 8 { - return commit[:8] - } - return commit -} - -// whereToPublish is the artifact store this builder uses. -// -// **Preferably from the mesh.** A builder that is a module requires an artifact store, and the -// mesh writes it a file saying which machine answers that and on what port — the same binding any -// consumer of any provision gets. Reading it means the address is not a setting somebody keeps in -// step by hand, and moving the store is an ordinary reassignment rather than an edit on every -// build machine. -// -// The environment variable remains for a builder run by a person, which is how this started and -// how it is still run while being developed. -func whereToPublish() (string, error) { - binding := strings.TrimSpace(os.Getenv("MESH_BINDING")) - if binding == "" { - registry := strings.TrimSpace(os.Getenv("MESH_REGISTRY")) - if registry == "" { - return "", fmt.Errorf("neither MESH_BINDING nor MESH_REGISTRY: a built artifact " + - "nobody can fetch is not built") - } - return registry, nil - } - - raw, err := os.ReadFile(binding) - if err != nil { - return "", fmt.Errorf("cannot read what the mesh said about the artifact store: %w", err) - } - var told struct { - From string `json:"from"` - At string `json:"at"` - Serves map[string]any `json:"serves"` - } - if err := json.Unmarshal(raw, &told); err != nil { - return "", fmt.Errorf("%s is not a binding: %w", binding, err) - } - if told.At == "" { - // The provider is not on the private network, so there is no name to reach it by. Said - // rather than falling back to the machine's own name, which would publish to a store on - // the wrong machine and be found out much later. - return "", fmt.Errorf( - "%s says the artifact store is on %q and gives no address for it", binding, told.From) - } - port, ok := told.Serves["port"] - if !ok { - return "", fmt.Errorf("%s says nothing about which port the artifact store answers on", - binding) - } - return fmt.Sprintf("%s:%v", told.At, port), nil -} - -// brokerFrom is where this builder connects, and with what. -// -// **Preferably from a file the mesh sealed to this machine.** A builder that is a module is given -// its credential the way every other module is given one: generated or accepted centrally, sealed -// to the machine, written by the host. Putting it in an environment variable instead would mean -// the one copy that matters passing through a terminal and a process listing. -// -// The variable remains for a builder run by a person. -func brokerFrom() (Credential, error) { - if path := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); path != "" { - raw, err := os.ReadFile(path) - if err != nil { - return Credential{}, fmt.Errorf("cannot read this builder's credential: %w", err) - } - said := strings.TrimSpace(string(raw)) - if said == "" { - // An empty credential file is a machine that will connect as nobody and be refused, - // with the reason three layers away. - return Credential{}, fmt.Errorf("%s is empty, so this builder has no credential", path) - } - var held Credential - if err := json.Unmarshal([]byte(said), &held); err == nil && held.URL != "" { - return held, nil - } - // A file holding only a URL, which is what a person writing one by hand produces. The - // broker is then verified against whatever this machine already trusts. - return Credential{URL: said}, nil - } - return Credential{}, fmt.Errorf( - "no MESH_BROKER_FILE: a build machine with no credential for the bus has nothing to build") -} - -// Credential is what a build machine is given so it can reach the broker. -// -// Two things, because reaching a broker over TLS needs both: who to connect as, and what to check -// the certificate against. A mesh's broker presents a certificate of the mesh's own, which is in -// no public trust store, so a URL alone can only connect to a broker somebody else vouches for. -// -// **The same shape a node gets, for the same reason** (novox/hq ADR 0004): the fingerprint travels -// out of band — here, sealed with the credential — and the endpoint is verified once at connect. -type Credential struct { - URL string `json:"url"` - Fingerprint string `json:"fingerprint,omitempty"` - // User and Password ride beside the address on the bus being built (design 25): a credential - // embedded in a URL leaks into every log line that prints a connection, so the mesh seals them - // as two fields and this machine joins them once, here, to dial. - User string `json:"user,omitempty"` - Password string `json:"password,omitempty"` - // Claims are the seats the module this credential was issued for claims, as the mesh writes - // them beside the credential (novox/hq ADR 0159). The first is the build role this machine - // serves; a credential naming none is from before claims travelled in it. - Claims []struct { - Seat string `json:"seat"` - } `json:"claims,omitempty"` -} - -// seatsClaimed is the seats the credential names, in order. -func (c Credential) seatsClaimed() []string { - out := make([]string, 0, len(c.Claims)) - for _, claim := range c.Claims { - out = append(out, claim.Seat) - } - return out -} - -// onTheNewBus is whether a credential is for the bus being built: its address says so, and the -// mesh only ever seals such a credential with the user and password beside it. -func (c Credential) onTheNewBus() bool { return strings.HasPrefix(strings.TrimSpace(c.URL), "nats://") } - -// natsURL is the address with this machine's credential in it, for the one dial that needs it. -func (c Credential) natsURL() string { - rest := strings.TrimPrefix(strings.TrimSpace(c.URL), "nats://") - if c.User == "" { - return "nats://" + rest - } - return "nats://" + c.User + ":" + c.Password + "@" + rest -} diff --git a/cmd/mesh-builder/once.go b/cmd/mesh-builder/once.go deleted file mode 100644 index 11df0bae..00000000 --- a/cmd/mesh-builder/once.go +++ /dev/null @@ -1,188 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "os" - "strings" - - "github.com/novox/mesh-controller/internal/builder" -) - -// buildOnce is the builder doing one build and stopping, with no broker and no mesh. -// -// **This is how a mesh is raised** (novox/hq ADR 0073). The installer carries this program and runs -// it once, before anything else exists, to produce the control plane from the same repository and -// path that every later rebuild of it will use. What raises the mesh is therefore the same thing -// that maintains it — not a second mechanism that has to be kept in step with the first and is -// exercised once per new mesh, which is how often enough to rot. -// -// It takes no work from a queue and answers nobody: there is no broker yet, and the only thing -// waiting for the answer is the installer that started it. So the result goes to standard output as -// JSON, which is what the installer reads. -// -// With no --registry it publishes nowhere and the image stays in this machine's container runtime, -// named by the digest of its own configuration (see builder.Local). That is the genesis case. With -// one, it publishes as it always does — the same command is how an operator builds a module by hand -// on a mesh that already exists. -func buildOnce(ctx context.Context, args []string) error { - set := flag.NewFlagSet("build", flag.ContinueOnError) - path := set.String("path", "", "the module's directory inside the repository (default: its root)") - ref := set.String("ref", "", "the commit to build; a branch is a moving target somebody else controls") - registry := set.String("registry", "", - "host:port to publish to. Without it the artifacts stay in this machine's container runtime, which is the genesis case") - workspace := set.String("workspace", "", "where to clone and build (default: a temporary directory)") - on := multiple(set, "on", - "a base this build stands on, as /=. Repeatable") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) != 1 { - return errors.New("mesh-builder build [--path P] [--ref COMMIT] [--registry HOST:PORT]") - } - repository := positionals[0] - - where := *workspace - if where == "" { - where = os.TempDir() + "/mesh-builder-once" - } - - // What this build may stand on, said on the command line because there is no mesh to ask. - // - // **This is the genesis case and it is meant to be awkward.** On a running mesh the control - // plane answers this, because only it knows what this mesh holds. Here nothing has been built - // yet, so whoever runs this says what to use — and for the control plane, which is the one - // module raised before anything else exists, the answer is ordinarily nothing at all. - bases := map[string]string{} - for _, pair := range *on { - key, reference, found := strings.Cut(pair, "=") - if !found || key == "" || reference == "" { - return fmt.Errorf( - "--on takes /=, and %q is not that", pair) - } - bases[key] = reference - } - - // Local unless told otherwise, because the moment this exists for has nowhere to publish. A - // default pointing at a registry would mean genesis failing at a push to something that is not - // there yet, one step away from the thing that could explain it. - var publisher builder.Publisher = builder.Local{Run: builder.Command} - if *registry != "" { - publisher = builder.Registry{Address: *registry, Run: builder.Command} - } - - fmt.Fprintf(os.Stderr, "building %s", repository) - if *path != "" { - fmt.Fprintf(os.Stderr, " at %s", *path) - } - if *ref != "" { - fmt.Fprintf(os.Stderr, " at %s", *ref) - } - fmt.Fprintln(os.Stderr) - - machine := os.Getenv("MESH_NODE") - if machine == "" { - machine, _ = os.Hostname() - } - npmrc, err := packagesFrom(machine) - if err != nil { - return err - } - fmt.Fprintf(os.Stderr, " [packages] %s\n", saysRegistry(npmrc)) - built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc, - forgeFrom(), - func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) }) - if buildErr != nil { - return buildErr - } - - // To standard output, and everything else to standard error, so the caller can read this - // without having to separate it from progress. - out := onceResult{ - Module: built.Manifest.Module, - Commit: built.Commit, - Repository: repository, - Path: *path, - Ref: *ref, - Manifest: built.Manifest, - Against: built.Against, - Source: built.Source, - } - for _, r := range built.Read { - out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref}) - } - for _, made := range built.Built { - out.Made = append(out.Made, madeArtifact{Name: made.Name, Kind: made.Kind, Reference: made.Reference}) - } - body, err := json.MarshalIndent(out, "", " ") - if err != nil { - return err - } - fmt.Println(string(body)) - fmt.Fprintf(os.Stderr, " built %s from %s\n", built.Manifest.Module, short(built.Commit)) - return nil -} - -// onceResult is what one build reports to whoever started it. -// -// The same fields the mesh records for a build, so a reader comparing a genesis build against an -// ordinary one is comparing the same thing said the same way. -type onceResult struct { - Module string `json:"module"` - Commit string `json:"commit"` - Repository string `json:"repository"` - Path string `json:"path,omitempty"` - Ref string `json:"ref,omitempty"` - Manifest any `json:"manifest"` - Made []madeArtifact `json:"made"` - Against []string `json:"against,omitempty"` - Read []readRepository `json:"read,omitempty"` - Source string `json:"source-fingerprint,omitempty"` -} - -// readRepository is a repository this build read source from besides the module's own. -type readRepository struct { - Repository string `json:"repository"` - Ref string `json:"ref,omitempty"` -} - -type madeArtifact struct { - Name string `json:"name"` - Kind string `json:"kind"` - Reference string `json:"reference"` -} - -// parseAround lets flags appear on either side of the repository, because a person writing this by -// hand will put them wherever reads best and the standard parser stops at the first thing that is -// not a flag. The same helper the control plane's commands use, for the same reason. -func parseAround(set *flag.FlagSet, args []string) ([]string, error) { - var positionals []string - rest := args - for { - if err := set.Parse(rest); err != nil { - return nil, err - } - rest = set.Args() - if len(rest) == 0 { - return positionals, nil - } - positionals = append(positionals, rest[0]) - rest = rest[1:] - } -} - -// multiple is a flag that may be given more than once. -type repeated []string - -func (r *repeated) String() string { return strings.Join(*r, ", ") } -func (r *repeated) Set(v string) error { *r = append(*r, v); return nil } - -func multiple(set *flag.FlagSet, name, usage string) *[]string { - var values repeated - set.Var(&values, name, usage) - return (*[]string)(&values) -} diff --git a/cmd/mesh-builder/packages_test.go b/cmd/mesh-builder/packages_test.go deleted file mode 100644 index 2fb4a719..00000000 --- a/cmd/mesh-builder/packages_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/builder" -) - -// The build agent's package registry (novox/hq ADR 0336, issue 121): its npm-package-registry -// requirement is optional, so the binding may be absent, and on genesis's node the installer seals -// the forge it raised as one genesis document instead. - -// registryFiles names the three paths the build agent's environment always names, writing only those -// given. -func registryFiles(t *testing.T, binding, token, genesis string) { - t.Helper() - dir := t.TempDir() - for env, content := range map[string]string{ - "MESH_PACKAGE_BINDING": binding, - "MESH_NPM_TOKEN_FILE": token, - "MESH_GENESIS_PACKAGE_REGISTRY": genesis, - } { - path := filepath.Join(dir, env) - t.Setenv(env, path) - if content != "" { - if err := os.WriteFile(path, []byte(content), 0o600); err != nil { - t.Fatal(err) - } - } - } - for _, env := range []string{"MESH_NPM_REGISTRY", "MESH_NPM_TOKEN", "MESH_NPM_USER", "MESH_NPM_SCOPE"} { - t.Setenv(env, "") - } -} - -const ( - aBinding = `{"from":"gitea","at":"anchor.mesh","as":"anchor-agent", - "serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}` - aGenesisDocument = `{"scheme":"http","at":"127.0.0.1","port":3100,"npm-path":"/api/packages/novox/npm/", - "as":"mesh-builder","password":"genesis-secret"}` -) - -func TestUntilTheRequirementBindsTheGenesisDocumentIsRead(t *testing.T) { - registryFiles(t, "", "", aGenesisDocument) - n, err := packagesFrom("anchor") - if err != nil { - t.Fatalf("an absent binding beside a genesis document was refused: %v", err) - } - if n.From != builder.FromGenesis || n.Registry != "http://127.0.0.1:3100/api/packages/novox/npm/" || - n.Username != "mesh-builder" || n.Password != "genesis-secret" || n.Scope != "@novox" { - t.Fatalf("the genesis document was not what the registry came from: %+v", n) - } - file, err := n.File() - if err != nil || !strings.Contains(file, "//127.0.0.1:3100/api/packages/novox/npm/:username=mesh-builder") { - t.Fatalf("the .npmrc is not written from the genesis document: %v\n%s", err, file) - } - if got := saysRegistry(n); !strings.Contains(got, "genesis document") { - t.Errorf("the log does not say the genesis document: %s", got) - } -} - -func TestARealBindingWinsOverTheGenesisDocument(t *testing.T) { - registryFiles(t, aBinding, "bound-secret\n", aGenesisDocument) - n, err := packagesFrom("anchor") - if err != nil { - t.Fatal(err) - } - if n.From != builder.FromBinding || n.Registry != "http://anchor.mesh:3000/api/packages/novox/npm/" || - n.Username != "anchor-agent" || n.Password != "bound-secret" { - t.Fatalf("the binding did not win over the genesis document: %+v", n) - } - if got := saysRegistry(n); !strings.Contains(got, "binding") { - t.Errorf("the log does not say the binding: %s", got) - } -} - -func TestWithNeitherTheAgentHasNoRegistryAndSaysWhere(t *testing.T) { - registryFiles(t, "", "", "") - n, err := packagesFrom("workstation") - if err != nil { - t.Fatalf("an unbound optional requirement was a fault: %v", err) - } - if n.Enabled() || n.Machine != "workstation" { - t.Fatalf("a registry came from nowhere: %+v", n) - } - got := saysRegistry(n) - for _, want := range []string{"npm-package-registry", "unbound", "workstation"} { - if !strings.Contains(got, want) { - t.Errorf("the log does not say %q: %s", want, got) - } - } -} - -func TestAGenesisDocumentMissingAFieldIsRefusedByName(t *testing.T) { - registryFiles(t, "", "", `{"at":"127.0.0.1","port":3000,"npm-path":"/api/packages/novox/npm/","as":"mesh-builder"}`) - if _, err := packagesFrom("anchor"); err == nil || !strings.Contains(err.Error(), "password") { - t.Fatalf("a genesis document with no password was taken: %v", err) - } -} diff --git a/cmd/mesh-builder/ready.go b/cmd/mesh-builder/ready.go deleted file mode 100644 index 09b770a6..00000000 --- a/cmd/mesh-builder/ready.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "os" - "path/filepath" -) - -// **The build agent says it is ready** (novox/hq issue 544). It listens on nothing, so no http or tcp -// check can see it work; it writes a mark once it is bound to its worker and pulls build work, and its -// container's health check runs `mesh-builder ready`, which passes only while the mark is there. Without -// it the node-engine could judge the agent only alive, and every walk that moved it waited out the whole -// grace on each machine. -// -// The mark lives in the container's own temporary directory, never in the workspace the machine keeps: -// a new container starts without it, and a restarted one removes it before it binds again. -func readyMark() string { return filepath.Join(os.TempDir(), "mesh-builder.ready") } - -// notReady removes the mark: a holder starting is not taking work until it says so again. -func notReady() error { - if err := os.Remove(readyMark()); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("cannot remove the mark that this agent takes build work (%s): %w", readyMark(), err) - } - return nil -} - -// taking writes the mark. A mark that cannot be written is said, and the agent goes on building: its -// check then says it is not ready, which is the truth the gate should hear. -func taking(seat string) { - if err := os.WriteFile(readyMark(), []byte(seat+"\n"), 0o644); err != nil { - fmt.Fprintf(os.Stderr, "cannot mark that this agent takes build work (%s): %v\n", readyMark(), err) - return - } - fmt.Fprintf(os.Stderr, "taking build work from %s\n", seat) -} - -// ready is the health check: nil when this container's agent is bound and pulling build work. -func ready() error { - if _, err := os.Stat(readyMark()); err != nil { - return fmt.Errorf("not taking build work yet: %s is not there", readyMark()) - } - return nil -} diff --git a/cmd/mesh-builder/ready_test.go b/cmd/mesh-builder/ready_test.go deleted file mode 100644 index 4e96679a..00000000 --- a/cmd/mesh-builder/ready_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "os" - "testing" -) - -// The health check passes only between the agent saying it takes work and its next start (novox/hq -// issue 544). -func TestTheAgentIsReadyOnlyOnceItTakesWork(t *testing.T) { - t.Setenv("TMPDIR", t.TempDir()) - if err := ready(); err == nil { - t.Fatal("an agent that never took work is ready") - } - taking("mesh-build-machine") - if err := ready(); err != nil { - t.Fatalf("an agent taking work is not ready: %v", err) - } - // Started again, it is not ready until it binds again. - if err := notReady(); err != nil { - t.Fatal(err) - } - if err := ready(); err == nil { - t.Fatal("an agent starting again is ready before it takes work") - } - if err := notReady(); err != nil { - t.Fatalf("removing a mark that is not there failed: %v", err) - } - if _, err := os.Stat(readyMark()); err == nil { - t.Fatal("the mark is still there") - } -} diff --git a/cmd/mesh-builder/seat_test.go b/cmd/mesh-builder/seat_test.go deleted file mode 100644 index 85186465..00000000 --- a/cmd/mesh-builder/seat_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "encoding/json" - "testing" - - "github.com/novox/mesh-controller/internal/link" -) - -// The seat a build machine serves comes from its credential (novox/hq ADR 0190 handover). -func TestTheCredentialSaysWhichBuildRoleThisMachineServes(t *testing.T) { - var held Credential - if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.builder","password":"x", - "claims":[{"seat":"mesh-build-machine","scope":"mesh","serves":[]}]}`), &held); err != nil { - t.Fatal(err) - } - if got := link.BuildSeatClaimed(held.seatsClaimed()); got != "mesh-build-machine" { - t.Errorf("the old builder's credential serves %q", got) - } - var bare Credential - if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.build-agent","password":"x"}`), &bare); err != nil { - t.Fatal(err) - } - if got := link.BuildSeatClaimed(bare.seatsClaimed()); got != link.TheBuildMachine { - t.Errorf("a credential without claims serves %q, want %s", got, link.TheBuildMachine) - } -} diff --git a/cmd/mesh-builder/stdout_test.go b/cmd/mesh-builder/stdout_test.go deleted file mode 100644 index f33e9401..00000000 --- a/cmd/mesh-builder/stdout_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "os" - "strings" - "testing" -) - -// **The genesis path writes its result to stdout and nothing else there.** The installer captures -// stdout alone and parses it as JSON; one stray log line makes that fail with "invalid character" -// — exactly what a builder fmt.Printf caused, breaking a build that had worked. This keeps -// diagnostics on stderr so a future print cannot repeat it silently. -func TestBuilderDiagnosticsStayOffStdout(t *testing.T) { - allowed := map[string]bool{ - "string(body)": true, // once.go: the result JSON, which IS stdout - "version)": true, // --version - `"stopping")`: true, // the loop.s shutdown line - "usage)": true, // --help text, for a human - "said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335) - } - for _, file := range []string{"once.go", "main.go"} { - src, err := os.ReadFile(file) - if err != nil { - t.Fatal(err) - } - for i, raw := range strings.Split(string(src), "\n") { - line := strings.TrimSpace(raw) - if !strings.HasPrefix(line, "fmt.Printf(") && - !strings.HasPrefix(line, "fmt.Println(") && - !strings.HasPrefix(line, "fmt.Print(") { - continue - } - ok := false - for token := range allowed { - if strings.Contains(line, token) { - ok = true - } - } - if !ok { - t.Errorf("%s:%d writes a diagnostic to stdout, which the installer parses as JSON:\n %s", - file, i+1, line) - } - } - } -} diff --git a/cmd/mesh-builder/tried_test.go b/cmd/mesh-builder/tried_test.go deleted file mode 100644 index 06aaf706..00000000 --- a/cmd/mesh-builder/tried_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/builder" -) - -// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own -// check's verdict, how long it took, and each line naming what failed. -func TestATrysOutcomeNamesWhatFailed(t *testing.T) { - tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second, - Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"}, - Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", - Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}}) - if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" || - tried.Summary != "its merge-check.sh failed: TestX" { - t.Fatalf("a failed try: %+v", tried) - } - // A head that does not merge fails both layers: the check's own word. - conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"}, - Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}}) - if conflict.Verdict != "fail" || conflict.Summary != "does not merge" { - t.Fatalf("a try that does not merge: %+v", conflict) - } - // No verdict is an error, never a pass. - if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" { - t.Fatalf("a try with no verdict: %+v", none) - } -} diff --git a/cmd/mesh-builder/where_test.go b/cmd/mesh-builder/where_test.go deleted file mode 100644 index 5a55bdb7..00000000 --- a/cmd/mesh-builder/where_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package main - -import ( - "crypto/ed25519" - "crypto/rand" - "crypto/sha256" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/hex" - "math/big" - "net" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/broker" -) - -// Where a builder publishes. -// -// Preferably from the mesh: a builder that is a module requires an artifact store, and the mesh -// writes it a binding saying which machine answers and on what port. Reading it means the address -// is not a setting somebody keeps in step by hand. - -func binding(t *testing.T, body string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "artifact-store.json") - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - return path -} - -func TestTheMeshSaysWhereToPublish(t *testing.T) { - t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"provision":"artifact-store", - "from":"anchor","at":"anchor.internal","serves":{"port":5000,"scheme":"http"}}`)) - where, err := whereToPublish() - if err != nil { - t.Fatal(err) - } - if where != "anchor.internal:5000" { - t.Fatalf("got %q", where) - } -} - -func TestABindingWithNoAddressIsRefused(t *testing.T) { - // The provider is not on the private network, so there is no name to reach it by. Falling - // back to anything would publish to a store on the wrong machine and be found out much later. - t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"anchor","serves":{"port":5000}}`)) - _, err := whereToPublish() - if err == nil { - t.Fatal("a binding with nowhere to reach was accepted") - } - if !strings.Contains(err.Error(), "anchor") { - t.Fatalf("the failure does not name the machine: %v", err) - } -} - -func TestABindingWithNoPortIsRefused(t *testing.T) { - t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"a","at":"a.internal","serves":{}}`)) - if _, err := whereToPublish(); err == nil { - t.Fatal("a binding saying nothing about a port was accepted") - } -} - -func TestTheVariableStillWorksForABuilderRunByAPerson(t *testing.T) { - // Which is how this started and how it is still run while being developed. - t.Setenv("MESH_BINDING", "") - t.Setenv("MESH_REGISTRY", "127.0.0.1:5000") - where, err := whereToPublish() - if err != nil { - t.Fatal(err) - } - if where != "127.0.0.1:5000" { - t.Fatalf("got %q", where) - } -} - -func TestNeitherIsRefusedRatherThanGuessed(t *testing.T) { - t.Setenv("MESH_BINDING", "") - t.Setenv("MESH_REGISTRY", "") - if _, err := whereToPublish(); err == nil { - t.Fatal("a builder with nowhere to publish reported somewhere") - } -} - -func TestTheCredentialComesFromAFileTheMeshSealed(t *testing.T) { - // A builder that is a module is given its credential the way every module is: sealed to the - // machine and written by the host. An environment variable instead would put the one copy - // that matters through a terminal and a process listing. - path := filepath.Join(t.TempDir(), "broker") - if err := os.WriteFile(path, []byte("amqps://a-builder:secret@broker.internal:5671/\n"), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv("MESH_BROKER_FILE", path) - t.Setenv("MESH_BROKER_AMQP", "amqp://should-not-be-used@nowhere/") - - got, err := brokerFrom() - if err != nil { - t.Fatal(err) - } - if got.URL != "amqps://a-builder:secret@broker.internal:5671/" { - t.Fatalf("got %q", got.URL) - } -} - -// The credential the mesh seals carries what to check the broker's certificate against, because a -// mesh's broker presents a certificate of the mesh's own and no public trust store has it. A URL -// alone can only reach a broker somebody else vouches for. -func TestTheSealedCredentialCarriesWhatVerifiesTheBroker(t *testing.T) { - path := filepath.Join(t.TempDir(), "broker") - if err := os.WriteFile(path, []byte( - `{"url":"amqps://a-builder:secret@broker.internal:5671/","fingerprint":"abc123"}`), - 0o600); err != nil { - t.Fatal(err) - } - t.Setenv("MESH_BROKER_FILE", path) - t.Setenv("MESH_BROKER_AMQP", "") - - got, err := brokerFrom() - if err != nil { - t.Fatal(err) - } - if got.URL != "amqps://a-builder:secret@broker.internal:5671/" { - t.Fatalf("the url was lost: %q", got.URL) - } - if got.Fingerprint != "abc123" { - t.Fatal("the builder was given nothing to check the broker against, so it can only " + - "connect to a broker some public authority vouches for") - } -} - -func TestAnEmptyCredentialFileIsRefused(t *testing.T) { - // Otherwise the builder connects as nobody and is refused, with the reason three layers away. - path := filepath.Join(t.TempDir(), "broker") - if err := os.WriteFile(path, []byte("\n"), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv("MESH_BROKER_FILE", path) - t.Setenv("MESH_BROKER_AMQP", "") - if _, err := brokerFrom(); err == nil { - t.Fatal("an empty credential was accepted") - } -} - -func TestABuilderWithNoCredentialAtAllSaysSo(t *testing.T) { - t.Setenv("MESH_BROKER_FILE", "") - t.Setenv("MESH_BROKER_AMQP", "") - if _, err := brokerFrom(); err == nil { - t.Fatal("a builder with no broker reported one") - } -} - -// The pin is compared in the spelling the mesh writes it. -// -// A bare digest against a written fingerprint never matches, and the failure is indistinguishable -// from being pointed at the wrong broker — which is the one thing this check exists to report -// truthfully. It cost a lab run. -func TestThePinIsComparedInTheSpellingTheMeshWritesIt(t *testing.T) { - certificate, key := aServerCertificate(t) - der := certificate.Certificate[0] - sum := sha256.Sum256(der) - written := "sha256:" + hex.EncodeToString(sum[:]) - - listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{ - Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS12, - }) - if err != nil { - t.Fatal(err) - } - defer listener.Close() - go func() { - for { - conn, err := listener.Accept() - if err != nil { - return - } - _ = conn.(*tls.Conn).Handshake() - conn.Close() - } - }() - _ = key - - // The pin the mesh wrote must be accepted. - if err := handshakeWith(listener.Addr().String(), written); err != nil { - t.Fatalf("the broker this builder was told about was refused: %v", err) - } - // And a different one refused, or the check reports nothing. - other := "sha256:" + strings.Repeat("ab", 32) - if err := handshakeWith(listener.Addr().String(), other); err == nil { - t.Fatal("a broker this builder was not told about was accepted") - } -} - -// handshakeWith runs the pin check the builder dials with against an address. -func handshakeWith(address, pin string) error { - conn, err := tls.Dial("tcp", address, broker.PinnedToFingerprint(pin)) - if err != nil { - return err - } - return conn.Close() -} - -func aServerCertificate(t *testing.T) (tls.Certificate, ed25519.PrivateKey) { - t.Helper() - public, private, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - t.Fatal(err) - } - template := &x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{CommonName: "a broker"}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(time.Hour), - IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, - } - der, err := x509.CreateCertificate(rand.Reader, template, template, public, private) - if err != nil { - t.Fatal(err) - } - return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: private}, private -} diff --git a/internal/gosource/gosource_test.go b/internal/gosource/gosource_test.go index 9353f13c..cba556ba 100644 --- a/internal/gosource/gosource_test.go +++ b/internal/gosource/gosource_test.go @@ -184,10 +184,6 @@ func TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn(t *testing.T) { if err != nil { t.Fatal(err) } - builder, err := GoBuildSource(root, "cmd/mesh-builder") - if err != nil { - t.Fatal(err) - } for _, c := range []struct { entries []string name string @@ -203,11 +199,7 @@ func TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn(t *testing.T) { {controller, "the controller", "internal/conditions/condition.go", true}, {controller, "the controller", "internal/inventory/migrations/0001-nodes.sql", true}, {controller, "the controller", "modules/route-proxy/cmd/route-proxy/main.go", false}, - {controller, "the controller", "cmd/mesh-builder/main.go", false}, {controller, "the controller", "README.md", false}, - {builder, "the build seat's program", "cmd/mesh-builder/main.go", true}, - {builder, "the build seat's program", "internal/conditions/condition.go", false}, - {builder, "the build seat's program", "cmd/mesh-controller/main.go", false}, } { if SourceHolds(c.entries, c.file) != c.held { t.Errorf("%s: %s held %v, wanted %v", c.name, c.file, !c.held, c.held) diff --git a/modules/build-agent/Dockerfile b/modules/build-agent/Dockerfile deleted file mode 100644 index 05eff017..00000000 --- a/modules/build-agent/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -ARG GO_BASE -ARG ALPINE_BASE -# build-agent's image: the build machine itself, compiled into a container (novox/hq ADR 0190). -# -# **The source sits beside this module.** The program, cmd/mesh-builder with internal/builder and -# internal/catalogue, lives in this same repository, mesh-controller, which is also the build context -# (declared under build.artifacts[].context; a module lives whole in one repository, novox/hq ADR 0420), -# so this Dockerfile compiles ./cmd/mesh-builder from the repository root. The build seat hands it only -# that program's build source (novox/hq ADR 0267, the artifact's `compiles`): a merge elsewhere in the -# repository does not move this image. -FROM ${GO_BASE} AS build -WORKDIR /src -COPY go.mod go.sum ./ -RUN go mod download -COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -buildvcs=false -ldflags=-buildid= -o /mesh-builder ./cmd/mesh-builder - -# Unlike mesh-controller's own FROM scratch (ADR 0006: nothing to audit but one binary), the build -# machine's whole job is shelling out to git and docker — it needs a real userland to do that in, -# not a second copy of either tool vendored into this image. apk installs both from the base's own -# packages, not fetched on its own at build time. -FROM ${ALPINE_BASE} -RUN apk add --no-cache docker-cli git -COPY --from=build /mesh-builder /usr/local/bin/mesh-builder -ENTRYPOINT ["/usr/local/bin/mesh-builder"] diff --git a/modules/build-agent/README.md b/modules/build-agent/README.md deleted file mode 100644 index 196544d5..00000000 --- a/modules/build-agent/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# build-agent - -The mesh's build machine as a role every machine can hold (novox/hq ADR 0190). It holds the node seat -`node-build-agent`: every holder pulls one build at a time from the role's one work queue when it is -idle, so a tier of many images is built by as many machines as hold the seat and are online, and a -machine that is off builds nothing and blocks nothing. The controller asks the role, never a machine; -the outcome names the machine that built it. - -What a holding machine needs is what the builder always needed, said here once: a container runtime -(the socket is mounted), the artifact store as a provision, a workspace, and the bus credential. The code is `cmd/mesh-builder`, in this same repository beside the module, compiled from -the repository's main (`build.artifacts[].context`). - -Assign it to every machine with a container runtime. It replaces `builder`, the one-holder form of the -same thing; retire that once this is assigned where it was. - -A send that replaces the agent's container does not throw away the build it runs: the container declares -`busy` as `/usr/local/bin/mesh-builder busy` (novox/hq ADR 0335). The node-engine asks it before replacing it; from that -ask the agent takes no new build, the node-engine leaves it running while it says a build is in hand, and -replaces it once it says idle, or after 15 minutes at the latest. - -## The package registry (novox/hq ADR 0336) - -The package registry (`npm-package-registry`, provided by the gitea module) is an optional requirement. -Genesis assigns the build agent before anything provides it, so the agent is assignable on a mesh with no -registry, and `mesh-controller.plan` and `node` list the requirement as unbound until the gitea module is -assigned; it then binds with no further act. - -Until it binds, the agent on genesis's node reads the `genesis-package-registry` own-secret, which the -installer seals to it at the packages step: the forge it raised, the port, the npm path, the account -`mesh-builder` and its password. Issued outside the mesh, so no other machine is given one. A real binding -wins over it, and each build's log says which the build resolved from. A build that needs the registry (a -`package` artifact, an image whose Dockerfile names `.npmrc`, a TypeScript artifact with an `@novox/` -dependency) on an agent with neither fails before building, naming the machine. Go artifacts and images -that do not name `.npmrc` build without it. `manifest_test.go` holds the manifest to this. diff --git a/modules/build-agent/files/build-agent.env b/modules/build-agent/files/build-agent.env deleted file mode 100644 index 819098b4..00000000 --- a/modules/build-agent/files/build-agent.env +++ /dev/null @@ -1,9 +0,0 @@ -MESH_BROKER_FILE=/run/mesh/broker -MESH_NODE=${machine:name} -MESH_REGISTRY=${bound:artifact-store:at}:${bound:artifact-store:port} -MESH_PACKAGE_BINDING=/run/mesh/package-registry.json -MESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret -MESH_GENESIS_PACKAGE_REGISTRY=/run/mesh/genesis-package-registry.json -MESH_WORKSPACE=${dir:workspace} -MESH_JUDGE_CACHE=${dir:judge-cache} -MESH_BUILD_CGROUP_PARENT=meshbuild.slice diff --git a/modules/build-agent/files/meshbuild.slice b/modules/build-agent/files/meshbuild.slice deleted file mode 100644 index 6258a1af..00000000 --- a/modules/build-agent/files/meshbuild.slice +++ /dev/null @@ -1,10 +0,0 @@ -# Managed by the mesh (module build-agent). Replaced on every push; edit the module's settings instead. -# -# Everything a build or check starts on this machine runs in this slice (novox/hq issue 1798): the build -# agent starts every container with --cgroup-parent=meshbuild.slice. The weights only bind while the machine -# is busy, so an idle machine still builds at full speed; the memory limits are the part that always binds. -[Slice] -CPUWeight=${setting:cpu-weight} -IOWeight=${setting:io-weight} -MemoryHigh=${setting:memory-high} -MemoryMax=${setting:memory-max} diff --git a/modules/build-agent/go.mod b/modules/build-agent/go.mod deleted file mode 100644 index 8d3d54b0..00000000 --- a/modules/build-agent/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module build-agent - -go 1.22 diff --git a/modules/build-agent/manifest_test.go b/modules/build-agent/manifest_test.go deleted file mode 100644 index 06d0f7ff..00000000 --- a/modules/build-agent/manifest_test.go +++ /dev/null @@ -1,247 +0,0 @@ -// The build agent's manifest (novox/hq ADR 0336, issue 121). The build agent is assigned at genesis's -// step 13, before anything provides the package registry, so that requirement is optional -// (ADR 0330); until it binds, the agent on genesis's node reads the genesis document the installer -// seals to it at the packages step. The program is mesh-controller's cmd/mesh-builder; only the -// packaging lives here, and this is its test. -package buildagent - -import ( - "encoding/json" - "os" - "strings" - "testing" -) - -type manifest struct { - Reads []string `json:"reads"` - Requires []json.RawMessage `json:"requires"` - Binds map[string]string `json:"binds"` - Secrets map[string]string `json:"secrets"` - OwnSecrets map[string]json.RawMessage `json:"own-secrets"` - Resources []map[string]any `json:"resources"` -} - -func read(t *testing.T) manifest { - t.Helper() - raw, err := os.ReadFile("module.json") - if err != nil { - t.Fatal(err) - } - var m manifest - if err := json.Unmarshal(raw, &m); err != nil { - t.Fatal(err) - } - return m -} - -// requirement is one entry of `requires`: its name, and whether it is optional. -func requirements(t *testing.T, m manifest) map[string]bool { - t.Helper() - out := map[string]bool{} - for _, raw := range m.Requires { - var name string - if json.Unmarshal(raw, &name) == nil { - out[name] = false - continue - } - var long struct { - Name string `json:"name"` - Optional bool `json:"optional"` - } - if err := json.Unmarshal(raw, &long); err != nil { - t.Fatalf("requires: %s: %v", raw, err) - } - out[long.Name] = long.Optional - } - return out -} - -func TestThePackageRegistryIsOptionalAndTheArtifactStoreStaysHard(t *testing.T) { - r := requirements(t, read(t)) - optional, has := r["npm-package-registry"] - if !has || !optional { - t.Errorf("npm-package-registry is not an optional requirement (ADR 0336 §1): %v", r) - } - if optional, has := r["artifact-store"]; !has || optional { - t.Errorf("artifact-store is not a hard requirement: %v", r) - } -} - -// The binding and the credential of the registry are delivered only while it is bound: ADR 0330 §6 -// composes neither for an absent requirement, so no file of the module may name them through a -// placeholder unless it is itself only-when-bound. -func TestTheRegistrysBindsAndSecretsAreOnlyWhenBound(t *testing.T) { - m := read(t) - if m.Binds["npm-package-registry"] == "" || m.Secrets["npm-package-registry"] == "" { - t.Fatalf("the agent is not told where the registry is once it binds: binds %v, secrets %v", m.Binds, m.Secrets) - } - for _, r := range m.Resources { - if r["type"] != "file" { - continue - } - content := templateText(t, r) - if strings.Contains(content, "${bound:npm-package-registry:") || strings.Contains(content, "${secret:npm-package-registry}") { - if r["only-when-bound"] != "npm-package-registry" { - t.Errorf("%v names the registry's binding and is delivered while it is absent", r["id"]) - } - } - } -} - -// The genesis document is an own-secret issued outside the mesh — by the installer at genesis's packages -// step (ADR 0336 §3) — so the mesh never makes one of its own on any other machine, and the agent is -// told where it lands. -func TestTheGenesisDocumentIsDeclaredAndIssuedOutside(t *testing.T) { - m := read(t) - raw, has := m.OwnSecrets["genesis-package-registry"] - if !has { - t.Fatalf("the build agent declares no genesis-package-registry own-secret: %v", m.OwnSecrets) - } - var own struct { - Path string `json:"path"` - IssuedBy string `json:"issued-by"` - } - if err := json.Unmarshal(raw, &own); err != nil || own.IssuedBy != "outside" { - t.Fatalf("the genesis document is not issued outside the mesh: %s (%v)", raw, err) - } - if !strings.HasPrefix(own.Path, "${dir:mesh-state}/") { - t.Fatalf("the genesis document is not where the container reads /run/mesh: %s", own.Path) - } - inside := "/run/mesh/" + strings.TrimPrefix(own.Path, "${dir:mesh-state}/") - for _, r := range m.Resources { - if r["id"] == "agent-env" { - content := templateText(t, r) - if !strings.Contains(content, "MESH_GENESIS_PACKAGE_REGISTRY="+inside+"\n") { - t.Errorf("the agent is not told where the genesis document is (%s):\n%s", inside, content) - } - return - } - } - t.Fatal("the build agent has no environment file") -} - -// **The judge cache is the agent's own, beside its workspace and never in it** (novox/hq ADR 0358 §6, issue 704): a -// directory resource of its own, kept as the agent's data, mounted into the agent at the same path as on the host — -// the container runtime resolves a judge build's mounts on the host — and handed to the agent as MESH_JUDGE_CACHE. -func TestTheJudgeCacheIsADirectoryOfItsOwnMountedAtTheSamePath(t *testing.T) { - raw, err := os.ReadFile("module.json") - if err != nil { - t.Fatal(err) - } - var m struct { - Data struct { - Own []struct { - ID string `json:"id"` - Path string `json:"path"` - Class string `json:"class"` - } `json:"own"` - } `json:"data"` - Resources []struct { - ID string `json:"id"` - Type string `json:"type"` - Template string `json:"template"` - Volumes []string `json:"volumes"` - } `json:"resources"` - } - if err := json.Unmarshal(raw, &m); err != nil { - t.Fatal(err) - } - dir, data, env, mounted := false, false, false, false - for _, d := range m.Data.Own { - if d.ID == "judge-cache" && d.Path == "${dir:judge-cache}" && d.Class == "cache" { - data = true - } - } - for _, r := range m.Resources { - switch { - case r.ID == "judge-cache" && r.Type == "directory": - dir = true - case r.ID == "agent-env": - env = strings.Contains(templateText(t, map[string]any{"id": r.ID, "template": r.Template}), "MESH_JUDGE_CACHE=${dir:judge-cache}\n") - case r.ID == "server": - for _, v := range r.Volumes { - if v == "${dir:judge-cache}:${dir:judge-cache}" { - mounted = true - } - if strings.HasPrefix(v, "${dir:workspace}") && strings.Contains(v, "judge-cache") { - t.Errorf("the judge cache is mounted inside the workspace: %s", v) - } - } - } - } - if !dir || !data || !env || !mounted { - t.Fatalf("the judge cache: directory %v, data %v, env %v, mounted at the same path %v", dir, data, env, mounted) - } -} - -// A running check's watch reads the pull requests' keys in the gate's bucket and in mesh-delivery's, so it -// stays sighted whichever of them the keys are written to (novox/hq ADR 0450 §1, issue 1325 AC1). -func TestTheAgentReadsTheGatesPullKeysAndTodaysBoth(t *testing.T) { - got := map[string]bool{} - for _, r := range read(t).Reads { - got[r] = true - } - for _, want := range []string{"mesh-delivery-gate.pulls", "mesh-delivery.pulls"} { - if !got[want] { - t.Errorf("the build agent does not read %s: %v", want, read(t).Reads) - } - } -} - -// Builds and checks run below the machine's own work (novox/hq issue 1798): the module declares the slice -// every build container joins with its limits as settings that have defaults, and tells the agent its name. -func TestTheBuildSliceCarriesTheMachinesLimits(t *testing.T) { - raw, err := os.ReadFile("module.json") - if err != nil { - t.Fatal(err) - } - var m struct { - Config map[string]struct { - Default any `json:"default"` - } `json:"config"` - } - if err := json.Unmarshal(raw, &m); err != nil { - t.Fatal(err) - } - slice, err := os.ReadFile("files/meshbuild.slice") - if err != nil { - t.Fatal(err) - } - for key, property := range map[string]string{"cpu-weight": "CPUWeight", "io-weight": "IOWeight", "memory-high": "MemoryHigh", "memory-max": "MemoryMax"} { - if c, ok := m.Config[key]; !ok || c.Default == nil { - t.Errorf("1798/AC2: setting %s is missing or has no default", key) - } - if !strings.Contains(string(slice), property+"=${setting:"+key+"}") { - t.Errorf("1798/AC2: the slice does not set %s from %s", property, key) - } - } - var placed, running bool - for _, r := range read(t).Resources { - placed = placed || (r["type"] == "file" && r["path"] == "/etc/systemd/system/meshbuild.slice") - running = running || (r["type"] == "service" && r["unit"] == "meshbuild.slice" && r["state"] == "running") - } - if !placed || !running { - t.Errorf("1798/AC2: the slice must be placed (%v) and running (%v)", placed, running) - } - // 1798/AC3: a dash in a slice's name nests it in the slice named before the dash, and cgroup v2 weights only - // compare siblings, so the name has none and the slice sits at the top level beside user.slice. - for _, r := range read(t).Resources { - if r["type"] == "service" && r["unit"] == "meshbuild.slice" { - if strings.Contains(strings.TrimSuffix(r["unit"].(string), ".slice"), "-") { - t.Errorf("1798/AC3: slice name %v nests", r["unit"]) - } - // 1798/AC4: a changed limit reaches systemd only if the unit restarts when its file changes. - restart, _ := r["restart-on"].([]any) - if len(restart) != 1 || restart[0] != "build-slice" { - t.Errorf("1798/AC4: the slice unit does not restart on its file: %v", restart) - } - } - } - env, err := os.ReadFile("files/build-agent.env") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(env), "MESH_BUILD_CGROUP_PARENT=meshbuild.slice") { - t.Error("1798/AC1: the agent is not told the slice's name") - } -} diff --git a/modules/build-agent/module.json b/modules/build-agent/module.json deleted file mode 100644 index c25e711a..00000000 --- a/modules/build-agent/module.json +++ /dev/null @@ -1,190 +0,0 @@ -{ - "module": "build-agent", - "version": "1", - "slug": "agent", - "config": { - "cpu-weight": { - "type": "number", - "description": "The CPU weight of everything a build or check starts on this machine, from 1 to 10000 against 100 for an ordinary service or a login session: under load the builds get their share of that, on an idle machine they take what is free (novox/hq issue 1798). The default 20 is about a sixth of the CPU against one competing workload; a desktop may take 10. Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.", - "minimum": 1, - "maximum": 10000, - "default": 20 - }, - "io-weight": { - "type": "number", - "description": "The disk IO weight of everything a build or check starts, from 1 to 10000 against 100 for an ordinary service: the default 10 lets the machine's own work go first (novox/hq issue 1798). Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.", - "minimum": 1, - "maximum": 10000, - "default": 10 - }, - "memory-high": { - "type": "string", - "description": "The memory above which builds and checks are slowed and made to give memory back, a percentage of the machine's memory or a size (systemd's MemoryHigh): the default 50% leaves half the machine to what it runs (novox/hq issue 1798). Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.", - "pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]", - "default": "50%" - }, - "memory-max": { - "type": "string", - "description": "The memory builds and checks together may never exceed, a percentage or a size (systemd's MemoryMax); a build past it is killed by the kernel and fails, so keep it above memory-high: the default 70% (novox/hq issue 1798). Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.", - "pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]", - "default": "70%" - } - }, - "capabilities": [ - "container-runtime" - ], - "claims": [ - { - "name": "node-build-agent", - "scope": "node", - "serves": [ - "current", - "kill", - "pause", - "resume" - ] - } - ], - "reads": [ - "mesh-delivery-gate.pulls", - "mesh-delivery.pulls" - ], - "requires": [ - "artifact-store", - { - "name": "npm-package-registry", - "optional": true - } - ], - "binds": { - "npm-package-registry": "${dir:mesh-state}/package-registry.json" - }, - "secrets": { - "npm-package-registry": "${dir:mesh-state}/package-registry.secret" - }, - "own-secrets": { - "broker": "${dir:mesh-state}/broker", - "genesis-package-registry": { - "path": "${dir:mesh-state}/genesis-package-registry.json", - "issued-by": "outside" - } - }, - "data": { - "own": [ - { - "id": "workspace", - "path": "${dir:workspace}", - "class": "cache", - "why": "a build's working copy, cloned again for every build" - }, - { - "id": "judge-cache", - "path": "${dir:judge-cache}", - "class": "cache", - "why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 \u00a76)" - } - ] - }, - "resources": [ - { - "id": "mesh-state", - "type": "directory", - "mode": "0700", - "place": "mesh" - }, - { - "id": "workspace", - "type": "directory", - "mode": "0700" - }, - { - "id": "judge-cache", - "type": "directory", - "mode": "0700" - }, - { - "id": "build-slice", - "type": "file", - "path": "/etc/systemd/system/meshbuild.slice", - "mode": "0644", - "trusted": false, - "template": "files/meshbuild.slice" - }, - { - "id": "build-slice-unit", - "type": "service", - "unit": "meshbuild.slice", - "state": "running", - "restart-on": [ - "build-slice" - ] - }, - { - "id": "agent-env", - "type": "file", - "path": "${dir:mesh-state}/build-agent.env", - "mode": "0600", - "template": "files/build-agent.env" - }, - { - "id": "server", - "type": "container", - "name": "mesh-build-agent", - "artifact": "server", - "env-file": [ - "${dir:mesh-state}/build-agent.env" - ], - "volumes": [ - "${dir:mesh-state}:/run/mesh:ro", - "${dir:workspace}:${dir:workspace}", - "${dir:judge-cache}:${dir:judge-cache}", - "/var/run/docker.sock:/var/run/docker.sock" - ], - "restart-on": [ - "agent-env" - ], - "network": "host", - "health": { - "kind": "exec", - "command": "/usr/local/bin/mesh-builder ready", - "interval": "10s", - "timeout": "5s", - "looks": 3, - "grace": "60s" - }, - "busy": { - "command": "/usr/local/bin/mesh-builder busy" - } - } - ], - "build": { - "artifacts": [ - { - "name": "server", - "kind": "image", - "from": "Dockerfile", - "compiles": "cmd/mesh-builder", - "context": { - "seat": "git", - "repository": "novox/mesh-controller", - "ref": "main" - } - }, - { - "name": "files", - "kind": "archive", - "from": "files" - } - ], - "on": [ - { - "arg": "GO_BASE", - "image": "golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c" - }, - { - "arg": "ALPINE_BASE", - "image": "alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc" - } - ] - } -} diff --git a/modules/build-agent/templates_test.go b/modules/build-agent/templates_test.go deleted file mode 100644 index 896a9770..00000000 --- a/modules/build-agent/templates_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package buildagent - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" -) - -// The module's directory, where its manifest and its files/ are. -const moduleDir = "." - -// templateText is the text a file resource places on the machine: its template's, in the module's files/ -// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. -func templateText(t *testing.T, r map[string]any) string { - t.Helper() - name, _ := r["template"].(string) - if name == "" { - t.Fatalf("resource %v names no template", r["id"]) - } - raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) - if err != nil { - t.Fatalf("resource %v names %s: %v", r["id"], name, err) - } - return string(raw) -} - -// No file resource carries its text inline: each names its template in files/, and the build packs files/ -// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). -func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { - raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) - if err != nil { - t.Fatal(err) - } - var m struct { - Resources []map[string]any `json:"resources"` - Build struct { - Artifacts []map[string]any `json:"artifacts"` - } `json:"build"` - } - if err := json.Unmarshal(raw, &m); err != nil { - t.Fatal(err) - } - for _, r := range m.Resources { - if _, inline := r["content"]; inline { - t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) - } - if r["type"] == "file" { - templateText(t, r) - } - } - packs := 0 - for _, a := range m.Build.Artifacts { - if a["kind"] == "archive" && a["from"] == "files" { - packs++ - if a["name"] != "files" { - t.Errorf("the templates archive is named %v, not files", a["name"]) - } - } - } - if packs != 1 { - t.Errorf("the build packs files/ into %d archives, not one", packs) - } -}