Archived
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
13be75eb65 | ||
|
|
13711fb5b9 | ||
|
|
50c55a773e | ||
|
|
45a1783ba9 |
@@ -66,6 +66,11 @@ broker to take work from or a registry to publish into:
|
||||
|
||||
Without --registry the artifacts stay in this machine's container runtime, named by the
|
||||
digest of their own configuration. The result is printed as JSON.
|
||||
|
||||
mesh-builder ready
|
||||
|
||||
exits 0 once the agent running in this container is bound to its build work and pulls it, and
|
||||
1 before: its container's health check.
|
||||
`
|
||||
|
||||
func run() error {
|
||||
@@ -76,12 +81,18 @@ func run() error {
|
||||
return nil
|
||||
case "build":
|
||||
return buildOnce(context.Background(), os.Args[2:])
|
||||
case "ready":
|
||||
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
|
||||
return ready()
|
||||
default:
|
||||
fmt.Print(usage)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := notReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
credential, err := brokerFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -134,7 +145,8 @@ func run() error {
|
||||
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
|
||||
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
|
||||
h.removeLeftBehind()
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
|
||||
Taking: func() { taking(seat) }})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// **The build agent says it is ready** (novox/hq issue 544). It listens on nothing, so no http or tcp
|
||||
// check can see it work; it writes a mark once it is bound to its worker and pulls build work, and its
|
||||
// container's health check runs `mesh-builder ready`, which passes only while the mark is there. Without
|
||||
// it the node-engine could judge the agent only alive, and every walk that moved it waited out the whole
|
||||
// grace on each machine.
|
||||
//
|
||||
// The mark lives in the container's own temporary directory, never in the workspace the machine keeps:
|
||||
// a new container starts without it, and a restarted one removes it before it binds again.
|
||||
func readyMark() string { return filepath.Join(os.TempDir(), "mesh-builder.ready") }
|
||||
|
||||
// notReady removes the mark: a holder starting is not taking work until it says so again.
|
||||
func notReady() error {
|
||||
if err := os.Remove(readyMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the mark that this agent takes build work (%s): %w", readyMark(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// taking writes the mark. A mark that cannot be written is said, and the agent goes on building: its
|
||||
// check then says it is not ready, which is the truth the gate should hear.
|
||||
func taking(seat string) {
|
||||
if err := os.WriteFile(readyMark(), []byte(seat+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot mark that this agent takes build work (%s): %v\n", readyMark(), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "taking build work from %s\n", seat)
|
||||
}
|
||||
|
||||
// ready is the health check: nil when this container's agent is bound and pulling build work.
|
||||
func ready() error {
|
||||
if _, err := os.Stat(readyMark()); err != nil {
|
||||
return fmt.Errorf("not taking build work yet: %s is not there", readyMark())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The health check passes only between the agent saying it takes work and its next start (novox/hq
|
||||
// issue 544).
|
||||
func TestTheAgentIsReadyOnlyOnceItTakesWork(t *testing.T) {
|
||||
t.Setenv("TMPDIR", t.TempDir())
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent that never took work is ready")
|
||||
}
|
||||
taking("mesh-build-machine")
|
||||
if err := ready(); err != nil {
|
||||
t.Fatalf("an agent taking work is not ready: %v", err)
|
||||
}
|
||||
// Started again, it is not ready until it binds again.
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ready(); err == nil {
|
||||
t.Fatal("an agent starting again is ready before it takes work")
|
||||
}
|
||||
if err := notReady(); err != nil {
|
||||
t.Fatalf("removing a mark that is not there failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(readyMark()); err == nil {
|
||||
t.Fatal("the mark is still there")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// TestReplay201ThePlansSend replays novox/hq issue 201 (2026-10-02): two merges a minute apart made two plans; the
|
||||
// controller's plan rolled the newer controller onto the control node, and the node-engine's plan then sent that
|
||||
// machine the controller build before it, whose binary refused the seat row the newer one had widened. Here the
|
||||
// control node, anchor, was last sent the newer controller c2 while the mesh's record of the controller is the
|
||||
// build before it, c1, which passed its gate. The outcome asserted: a send that is no person's push — composed as a
|
||||
// plan's send composes it (keepingRecorded, what sendKeeps gives every send through sendToEach) — carries the
|
||||
// controller at c2, the build the machine was last sent, and not the older record; a person's push still carries
|
||||
// the record. Written with only what the controller had before its fix (mesh-controller #273) — the inventory's
|
||||
// build, gate, assignment and send records, keepingRecorded, planFor and declarationFor — so it is laid over the
|
||||
// commit before; the fix's own TestReplay201 reads what the fix added. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
func TestReplay201ThePlansSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
const older, newer = "c1aaaaaaaaaa", "c2bbbbbbbbbb"
|
||||
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
||||
record := func(commit string, asked time.Time) inventory.Build {
|
||||
t.Helper()
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
|
||||
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
|
||||
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// The older controller rolled out and passed its gate; the newer one was asked after it and registered.
|
||||
before := record(older, old)
|
||||
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: older,
|
||||
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record(newer, now)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-controller", BuiltFrom: newer, Head: newer, Asked: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The controller's plan sent anchor the newer controller.
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": newer}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// And the mesh's record of the controller is the build before it — whatever made it so.
|
||||
if err := inv.RestoreModule(ctx, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The controller each send to anchor carries: a plan's send keeps recorded builds (keepingRecorded, what
|
||||
// sendKeeps gives every send but a person's push); a person's push composes the mesh's record.
|
||||
controllerSent := func(sends string, keeping bool) string {
|
||||
t.Helper()
|
||||
c := ctx
|
||||
if keeping {
|
||||
c = keepingRecorded(ctx)
|
||||
}
|
||||
plan, settings, err := planFor(c, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatalf("%s to anchor did not compose: %v", sends, err)
|
||||
}
|
||||
declared, err := declarationFor(c, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("%s to anchor did not compose: %v", sends, err)
|
||||
}
|
||||
return declared.Builds["mesh-controller"]
|
||||
}
|
||||
|
||||
if got := controllerSent("a plan's send", true); got != newer {
|
||||
t.Fatalf("a plan's send carried anchor the controller %s, older than the %s anchor was last sent: the "+
|
||||
"older binary would read the row its successor widened", short(got), short(newer))
|
||||
}
|
||||
if got := controllerSent("a person's push", false); got != older {
|
||||
t.Fatalf("a person's push carried the controller %s, not the mesh's record %s", short(got), short(older))
|
||||
}
|
||||
}
|
||||
@@ -167,6 +167,10 @@ type MachineOptions struct {
|
||||
// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
|
||||
// already running finishes. Nil is never paused.
|
||||
Paused func() bool
|
||||
// Taking is called once, when this machine is bound to its worker and pulls work: from then on it
|
||||
// can take a build, which is what its container's health check reads (novox/hq issue 544). Nil is
|
||||
// told nothing.
|
||||
Taking func()
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
@@ -232,6 +236,9 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
"the mesh's to answer, not this machine's", m.seat, err)
|
||||
}
|
||||
m.sub = sub
|
||||
if m.opts.Taking != nil {
|
||||
m.opts.Taking()
|
||||
}
|
||||
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
|
||||
@@ -368,3 +368,36 @@ func TestNatsAMachineOnTheRetiredBuildRoleTakesThatRolesAsks(t *testing.T) {
|
||||
t.Errorf("the retired role's holder did not answer: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder says once that it takes work, and only once it is bound to its worker (novox/hq issue 544):
|
||||
// its container's health check reads that, so a holder that could not bind is never said ready.
|
||||
func TestNatsAHolderSaysItTakesWorkOnceItIsBound(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
said := make(chan struct{}, 4)
|
||||
machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Taking: func() { said <- struct{}{} }})
|
||||
defer machine.Close()
|
||||
go func() { _ = machine.Take(ctx, func(context.Context, Build) {}) }()
|
||||
select {
|
||||
case <-said:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a holder bound to its worker never said it takes work")
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("a holder said it takes work %d more time(s)", len(said))
|
||||
}
|
||||
|
||||
// A role the mesh raised no worker for cannot be bound, and its holder says nothing.
|
||||
bounded, end := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer end()
|
||||
other := MachineOverNATSWith(js, "anchor", "mesh-no-such-seat", MachineOptions{Taking: func() { said <- struct{}{} }})
|
||||
defer other.Close()
|
||||
if err := other.Take(bounded, func(context.Context, Build) {}); err == nil {
|
||||
t.Fatal("a machine with no worker took work")
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Fatal("a machine that could not bind said it takes work")
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+1
@@ -35,3 +35,4 @@
|
||||
./cmd/mesh-controller TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend
|
||||
./cmd/mesh-controller TestAGateJudgesTheOtherMovesBesideAKeptCore
|
||||
./cmd/mesh-controller TestReplay401
|
||||
./cmd/mesh-controller TestReplay201ThePlansSend
|
||||
|
||||
Reference in New Issue
Block a user