The controller asks the build role that has a holder, and hears both roles' outcomes (hq ADR 0190, the handover)

A controller that asked node-build-agent from its first run would queue every build where nothing
pulls, and the build that registers build-agent — the first holder — would be among them. So the
role is chosen at ask time from the catalogue: the current role when any assigned module claims it,
the retired one while only the builder does, the current one when neither. Outcomes are followed on
both seats, the controller may publish to both, and a build's log is read under whichever role did
it; a machine on the retired role is proven on the bus to take that role's asks. The switch order
is written where the role is named, and the retired half is marked for removal with the seat row.
This commit is contained in:
jochen
2026-10-03 02:51:03 +02:00
parent a5d6a1187c
commit ff5ef0ab60
9 changed files with 187 additions and 23 deletions
+55 -6
View File
@@ -430,11 +430,13 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti
}
fmt.Println()
ask, err := askOver(server)
seat := buildSeatHeld(ctx)
ask, err := askOverOn(seat)
if err != nil {
return err
}
defer ask.Close()
fmt.Printf(" of %s\n", seat)
if wait == 0 {
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
@@ -555,7 +557,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
}
defer server.Close()
ask, err := askOver(server)
ask, err := askOverOn(buildSeatHeld(ctx))
if err != nil {
return err
}
@@ -668,12 +670,56 @@ func heldBy(ctx context.Context) map[string]string {
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
// being built it dials, because a build request is a one-shot and holds nothing else.
func askOver(_ *link.Server) (link.Builders, error) {
func askOverOn(seat string) (link.Builders, error) {
address, err := broker.BusAddress()
if err != nil {
return nil, err
}
return link.BuildsOverNATS(address)
return link.BuildsOverNATSOn(address, seat)
}
// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190,
// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the
// moment the first build-agent is assigned — and a controller that asked the new role before then
// would queue work nothing takes, while the outcome that registers build-agent itself has to come
// from the old builder. When the catalogue cannot be read the current role is asked, said aloud.
func buildSeatHeld(ctx context.Context) string {
open, err := openStores(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n",
link.TheBuildMachine, err)
return link.TheBuildMachine
}
defer open.Close()
entries, err := open.inventory.Catalogued(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n",
link.TheBuildMachine, err)
return link.TheBuildMachine
}
return buildSeatAmong(entries)
}
// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any
// assigned module claims it; else the retired role while an assigned module still claims that; else
// the current role, which is where every ask goes once the handover is done.
func buildSeatAmong(entries []inventory.Entry) string {
heldBefore := false
for _, e := range entries {
if len(e.On) == 0 {
continue
}
if e.Manifest.ClaimsSeat(link.TheBuildMachine) {
return link.TheBuildMachine
}
if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) {
heldBefore = true
}
}
if heldBefore {
return link.TheBuildMachineBefore
}
return link.TheBuildMachine
}
// buildLog prints everything a build machine said about one build, read back from the bus.
@@ -693,10 +739,13 @@ func buildLog(ctx context.Context, id string) error {
}
defer js.Close()
sub, err := js.Context().PullSubscribe(link.BuildLog(id), "",
// Under whichever build role did it: a build asked of the retired role during the handover
// (ADR 0190) said its lines as that role's events, and a reader should not have to know which.
lines := link.BuildLogOf("*", id)
sub, err := js.Context().PullSubscribe(lines, "",
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
if err != nil {
return fmt.Errorf("cannot read %s from the bus: %w", link.BuildLog(id), err)
return fmt.Errorf("cannot read %s from the bus: %w", lines, err)
}
defer func() { _ = sub.Unsubscribe() }()
+43
View File
@@ -0,0 +1,43 @@
package main
import (
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
func claiming(module, seat string, on ...string) inventory.Entry {
return inventory.Entry{
Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}},
On: on,
}
}
// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired
// one while only the builder is assigned, the current one from the first build-agent on, and the
// current one when nothing holds either — where every ask goes once the handover is done.
func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) {
onlyTheBuilder := []inventory.Entry{
claiming("builder", link.TheBuildMachineBefore, "anchor"),
claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet
}
if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore {
t.Errorf("with only the builder assigned, asked %q", got)
}
bothHeld := []inventory.Entry{
claiming("builder", link.TheBuildMachineBefore, "anchor"),
claiming("build-agent", link.TheBuildMachine, "home-server"),
}
if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine {
t.Errorf("with a build-agent assigned anywhere, asked %q", got)
}
neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)}
if got := buildSeatAmong(neither); got != link.TheBuildMachine {
t.Errorf("with no holder of either, asked %q, want the current role", got)
}
if got := buildSeatAmong(nil); got != link.TheBuildMachine {
t.Errorf("an empty catalogue asks %q", got)
}
}
+4 -1
View File
@@ -113,7 +113,10 @@ type Principal struct {
// seatsTheControllerAsks are the roles the mesh's own flows submit work to. Named rather than
// derived from the seat set: the controller is not a module and declares no `uses`, so its side of a
// seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable.
var seatsTheControllerAsks = []string{"node-build-agent"}
// Both build roles while the handover runs (novox/hq ADR 0190): the controller asks whichever has a
// holder, and the retired one has one until build-agent replaces the builder. The second entry
// goes with the retired seat row.
var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
// controller may answer an enrolment is derived from the same constant the user is named from.
+5
View File
@@ -221,6 +221,11 @@ var ControllerFollows = []string{
// The forge's merges: what moved a source, so the mesh builds what that source produces
// without anybody telling it (novox/hq 04-ISSUES/131). Appended, because the index is a name.
moduleEventSubject("gitea", "pull.merged"),
// The retired build role's outcome too, while the handover runs (novox/hq ADR 0190): the one
// build machine keeps answering on its seat until build-agent replaces it, and the outcome that
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
// with the retired seat row.
seatEventSubject("mesh-build-machine", "built"),
}
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
+2 -2
View File
@@ -24,8 +24,8 @@ accounts {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] }
subscribe: { allow: ["$JS.API.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] }
subscribe: { allow: ["$JS.API.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
+15 -6
View File
@@ -23,12 +23,21 @@ import (
// builds, and the work shared among them (novox/hq ADR 0190). The name stays for every caller; what
// it names moved from the mesh's one build machine to whichever build agent is idle.
//
// **Switching a live mesh over, in order.** The old seat's stream and worker
// (SEAT_MESH_BUILD_MACHINE, SEAT_MESH_BUILD_MACHINE_worker) stay on the bus until removed by hand,
// and the builder module keeps draining them while it is assigned. From the moment a controller
// with this name runs, new asks go to node-build-agent and wait in its stream until some machine
// holds the seat. So: let the queued builds finish; roll this controller; register and assign
// build-agent to the machines that build; unassign builder and forget it and its seat's stream.
// **Switching a live mesh over, in order** — and why no step strands a build. The old seat's
// stream and worker (SEAT_MESH_BUILD_MACHINE, SEAT_MESH_BUILD_MACHINE_worker) stay on the bus until
// removed by hand, and the builder keeps draining them while it is assigned, because a machine
// serves the seat its credential claims (BuildSeatClaimed) and the controller asks the seat that
// has a holder (buildSeatAmong in the command) and hears both seats' outcomes:
//
// 1. Merge the controller and the host's first user list together; the new controller rolls and,
// seeing only the builder assigned, still asks mesh-build-machine — which the builder holds.
// 2. Merge the catalogue's build-agent; the builder builds it and the controller registers it.
// 3. On each machine that builds: `module issue build-agent --node <n>`, then `assign`, then
// `push`. The first holder appears, and from then on asks go to node-build-agent.
// 4. Unassign builder everywhere and `module forget` it.
// 5. By hand: delete SEAT_MESH_BUILD_MACHINE and its worker, drop the retired seat row and
// TheBuildMachineBefore with it, and the second entries in seatsTheControllerAsks and
// ControllerFollows.
const TheBuildMachine = "node-build-agent"
// TheBuildMachineBefore is the role a build was submitted to until ADR 0190: the mesh's one build
+24 -6
View File
@@ -25,16 +25,34 @@ import (
type natsBuilds struct {
js *broker.JetStream
owned bool
// seat is the build role asked: the one that has a holder (ADR 0190 handover), chosen by the
// controller from what is assigned, so an ask lands where a machine is pulling.
seat string
}
// BuildsOverNATS is the asking side on the bus being built. It dials, because the command that asks
// for a build is a one-shot and holds nothing else.
// BuildsOverNATS is the asking side on the bus being built, asking the current build role. It dials,
// because the command that asks for a build is a one-shot and holds nothing else.
func BuildsOverNATS(address string) (Builders, error) {
return BuildsOverNATSOn(address, TheBuildMachine)
}
// BuildsOverNATSOn is the asking side for one named build role — during the handover from the one
// build machine to build agents, the role that has a holder (ADR 0190).
func BuildsOverNATSOn(address, seat string) (Builders, error) {
js, err := broker.Dial(address)
if err != nil {
return nil, fmt.Errorf("cannot reach the bus at %s to ask for a build: %w", address, err)
}
return &natsBuilds{js: js, owned: true}, nil
return &natsBuilds{js: js, owned: true, seat: seat}, nil
}
// role is the seat asked: what the asker was made for, or the current build role for one made
// without saying (a test building the struct by hand).
func (b *natsBuilds) role() string {
if b.seat == "" {
return TheBuildMachine
}
return b.seat
}
func (b *natsBuilds) Close() {
@@ -51,7 +69,7 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
}
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if _, err := b.js.Context().Publish(BuildWork(), body, nats.Context(publish)); err != nil {
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
return fmt.Errorf("cannot submit a build: %w", err)
}
return nil
@@ -63,7 +81,7 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
// Subscribed before the ask, so an outcome cannot arrive before there is anywhere for it to
// land. Core, not the stream: the asker is waiting now, and the durable copy of this outcome is
// the same event on EVENTS, which the controller records.
outcomes, err := b.js.Conn().SubscribeSync(BuildOutcome())
outcomes, err := b.js.Conn().SubscribeSync(BuildOutcomeOf(b.role()))
if err != nil {
return BuildResult{}, fmt.Errorf("cannot listen for a build's outcome: %w", err)
}
@@ -80,7 +98,7 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
// be assumed, because nothing else will ever say so.
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if _, err := b.js.Context().Publish(BuildWork(), body, nats.Context(publish)); err != nil {
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
return BuildResult{}, fmt.Errorf("cannot submit a build: %w", err)
}
+36
View File
@@ -325,3 +325,39 @@ func TestNatsTwoMachinesShareTheWorkAndNeitherIsHandedMoreThanItCanTake(t *testi
case <-time.After(2 * time.Second):
}
}
// During the handover (ADR 0190) two build roles exist. A machine whose credential claims the retired
// one takes an ask published to that seat and answers as that seat; the asker of that seat hears it.
func TestNatsAMachineOnTheRetiredBuildRoleTakesThatRolesAsks(t *testing.T) {
js := aBusWithTheBuildRole(t)
seats := []broker.DeclaredSeat{{Name: TheBuildMachineBefore, Accepts: []string{"build"}, Emits: []string{"built"}}}
if err := broker.RaiseSeats(js, seats, map[string]broker.Holder{TheBuildMachineBefore: {Node: "anchor", Module: "builder"}}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") })
ctx, stop := context.WithCancel(context.Background())
defer stop()
machine := MachineOverNATSOn(js, "anchor", TheBuildMachineBefore)
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
_ = work.Began(ctx)
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, Repository: work.Request().Repository, On: "anchor", Commit: "abc"})
_ = work.Done()
})
}()
asker, err := BuildsOverNATSOn(os.Getenv("MESH_TEST_NATS"), TheBuildMachineBefore)
if err != nil {
t.Fatal(err)
}
defer asker.Close()
result, err := asker.Submit(ctx, BuildRequest{ID: "build-old-seat", Repository: "r"}, 20*time.Second)
if err != nil {
t.Fatal(err)
}
if result.On != "anchor" || result.ID != "build-old-seat" {
t.Errorf("the retired role's holder did not answer: %+v", result)
}
}
+3 -2
View File
@@ -223,10 +223,11 @@ func kindOfSubject(subject string) (string, bool) {
return KindCatchUp, true
case broker.ControllerFollows[3]:
return KindSourceMoved, true
case BuildOutcome():
case BuildOutcome(), BuildOutcomeOf(TheBuildMachineBefore):
// A build's outcome is the role's event now, so it arrives on the events stream rather than
// the control branch — and is acted on by the same handler, because what the controller does
// with it did not change (novox/hq ADR 0121).
// with it did not change (novox/hq ADR 0121). From either build role while the handover
// runs (ADR 0190): the old builder still answers on the retired seat until it is unassigned.
return KindBuilt, true
}
return "", false