diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 60695c3..56f7afb 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -430,11 +430,13 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti } fmt.Println() - ask, err := askOver(server) + seat := buildSeatHeld(ctx) + ask, err := askOverOn(seat) if err != nil { return err } defer ask.Close() + fmt.Printf(" of %s\n", seat) if wait == 0 { // Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the @@ -555,7 +557,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai } defer server.Close() - ask, err := askOver(server) + ask, err := askOverOn(buildSeatHeld(ctx)) if err != nil { return err } @@ -668,12 +670,56 @@ func heldBy(ctx context.Context) map[string]string { // **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus // the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus // being built it dials, because a build request is a one-shot and holds nothing else. -func askOver(_ *link.Server) (link.Builders, error) { +func askOverOn(seat string) (link.Builders, error) { address, err := broker.BusAddress() if err != nil { return nil, err } - return link.BuildsOverNATS(address) + return link.BuildsOverNATSOn(address, seat) +} + +// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190, +// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the +// moment the first build-agent is assigned — and a controller that asked the new role before then +// would queue work nothing takes, while the outcome that registers build-agent itself has to come +// from the old builder. When the catalogue cannot be read the current role is asked, said aloud. +func buildSeatHeld(ctx context.Context) string { + open, err := openStores(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n", + link.TheBuildMachine, err) + return link.TheBuildMachine + } + defer open.Close() + entries, err := open.inventory.Catalogued(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n", + link.TheBuildMachine, err) + return link.TheBuildMachine + } + return buildSeatAmong(entries) +} + +// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any +// assigned module claims it; else the retired role while an assigned module still claims that; else +// the current role, which is where every ask goes once the handover is done. +func buildSeatAmong(entries []inventory.Entry) string { + heldBefore := false + for _, e := range entries { + if len(e.On) == 0 { + continue + } + if e.Manifest.ClaimsSeat(link.TheBuildMachine) { + return link.TheBuildMachine + } + if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) { + heldBefore = true + } + } + if heldBefore { + return link.TheBuildMachineBefore + } + return link.TheBuildMachine } // buildLog prints everything a build machine said about one build, read back from the bus. @@ -693,10 +739,13 @@ func buildLog(ctx context.Context, id string) error { } defer js.Close() - sub, err := js.Context().PullSubscribe(link.BuildLog(id), "", + // Under whichever build role did it: a build asked of the retired role during the handover + // (ADR 0190) said its lines as that role's events, and a reader should not have to know which. + lines := link.BuildLogOf("*", id) + sub, err := js.Context().PullSubscribe(lines, "", nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone()) if err != nil { - return fmt.Errorf("cannot read %s from the bus: %w", link.BuildLog(id), err) + return fmt.Errorf("cannot read %s from the bus: %w", lines, err) } defer func() { _ = sub.Unsubscribe() }() diff --git a/cmd/mesh-controller/build_seat_test.go b/cmd/mesh-controller/build_seat_test.go new file mode 100644 index 0000000..e40ca0a --- /dev/null +++ b/cmd/mesh-controller/build_seat_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func claiming(module, seat string, on ...string) inventory.Entry { + return inventory.Entry{ + Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}}, + On: on, + } +} + +// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired +// one while only the builder is assigned, the current one from the first build-agent on, and the +// current one when nothing holds either — where every ask goes once the handover is done. +func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) { + onlyTheBuilder := []inventory.Entry{ + claiming("builder", link.TheBuildMachineBefore, "anchor"), + claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet + } + if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore { + t.Errorf("with only the builder assigned, asked %q", got) + } + bothHeld := []inventory.Entry{ + claiming("builder", link.TheBuildMachineBefore, "anchor"), + claiming("build-agent", link.TheBuildMachine, "home-server"), + } + if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine { + t.Errorf("with a build-agent assigned anywhere, asked %q", got) + } + neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)} + if got := buildSeatAmong(neither); got != link.TheBuildMachine { + t.Errorf("with no holder of either, asked %q, want the current role", got) + } + if got := buildSeatAmong(nil); got != link.TheBuildMachine { + t.Errorf("an empty catalogue asks %q", got) + } +} diff --git a/internal/broker/nats.go b/internal/broker/nats.go index b52cb34..025bcd5 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -113,7 +113,10 @@ type Principal struct { // seatsTheControllerAsks are the roles the mesh's own flows submit work to. Named rather than // derived from the seat set: the controller is not a module and declares no `uses`, so its side of a // seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable. -var seatsTheControllerAsks = []string{"node-build-agent"} +// Both build roles while the handover runs (novox/hq ADR 0190): the controller asks whichever has a +// holder, and the retired one has one until build-agent replaces the builder. The second entry +// goes with the retired seat row. +var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"} // enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the // controller may answer an enrolment is derived from the same constant the user is named from. diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 210359b..526cd65 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -221,6 +221,11 @@ var ControllerFollows = []string{ // The forge's merges: what moved a source, so the mesh builds what that source produces // without anybody telling it (novox/hq 04-ISSUES/131). Appended, because the index is a name. moduleEventSubject("gitea", "pull.merged"), + // The retired build role's outcome too, while the handover runs (novox/hq ADR 0190): the one + // build machine keeps answering on its seat until build-agent replaces it, and the outcome that + // registers build-agent itself comes from there. Appended, for the same reason as above; goes + // with the retired seat row. + seatEventSubject("mesh-build-machine", "built"), } // moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 3994266..ba18138 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,8 +24,8 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] } - subscribe: { allow: ["$JS.API.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] } + subscribe: { allow: ["$JS.API.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { diff --git a/internal/link/builds.go b/internal/link/builds.go index f43cf5b..5757671 100644 --- a/internal/link/builds.go +++ b/internal/link/builds.go @@ -23,12 +23,21 @@ import ( // builds, and the work shared among them (novox/hq ADR 0190). The name stays for every caller; what // it names moved from the mesh's one build machine to whichever build agent is idle. // -// **Switching a live mesh over, in order.** The old seat's stream and worker -// (SEAT_MESH_BUILD_MACHINE, SEAT_MESH_BUILD_MACHINE_worker) stay on the bus until removed by hand, -// and the builder module keeps draining them while it is assigned. From the moment a controller -// with this name runs, new asks go to node-build-agent and wait in its stream until some machine -// holds the seat. So: let the queued builds finish; roll this controller; register and assign -// build-agent to the machines that build; unassign builder and forget it and its seat's stream. +// **Switching a live mesh over, in order** — and why no step strands a build. The old seat's +// stream and worker (SEAT_MESH_BUILD_MACHINE, SEAT_MESH_BUILD_MACHINE_worker) stay on the bus until +// removed by hand, and the builder keeps draining them while it is assigned, because a machine +// serves the seat its credential claims (BuildSeatClaimed) and the controller asks the seat that +// has a holder (buildSeatAmong in the command) and hears both seats' outcomes: +// +// 1. Merge the controller and the host's first user list together; the new controller rolls and, +// seeing only the builder assigned, still asks mesh-build-machine — which the builder holds. +// 2. Merge the catalogue's build-agent; the builder builds it and the controller registers it. +// 3. On each machine that builds: `module issue build-agent --node `, then `assign`, then +// `push`. The first holder appears, and from then on asks go to node-build-agent. +// 4. Unassign builder everywhere and `module forget` it. +// 5. By hand: delete SEAT_MESH_BUILD_MACHINE and its worker, drop the retired seat row and +// TheBuildMachineBefore with it, and the second entries in seatsTheControllerAsks and +// ControllerFollows. const TheBuildMachine = "node-build-agent" // TheBuildMachineBefore is the role a build was submitted to until ADR 0190: the mesh's one build diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go index 9e158b6..c3b1011 100644 --- a/internal/link/builds_nats.go +++ b/internal/link/builds_nats.go @@ -25,16 +25,34 @@ import ( type natsBuilds struct { js *broker.JetStream owned bool + // seat is the build role asked: the one that has a holder (ADR 0190 handover), chosen by the + // controller from what is assigned, so an ask lands where a machine is pulling. + seat string } -// BuildsOverNATS is the asking side on the bus being built. It dials, because the command that asks -// for a build is a one-shot and holds nothing else. +// BuildsOverNATS is the asking side on the bus being built, asking the current build role. It dials, +// because the command that asks for a build is a one-shot and holds nothing else. func BuildsOverNATS(address string) (Builders, error) { + return BuildsOverNATSOn(address, TheBuildMachine) +} + +// BuildsOverNATSOn is the asking side for one named build role — during the handover from the one +// build machine to build agents, the role that has a holder (ADR 0190). +func BuildsOverNATSOn(address, seat string) (Builders, error) { js, err := broker.Dial(address) if err != nil { return nil, fmt.Errorf("cannot reach the bus at %s to ask for a build: %w", address, err) } - return &natsBuilds{js: js, owned: true}, nil + return &natsBuilds{js: js, owned: true, seat: seat}, nil +} + +// role is the seat asked: what the asker was made for, or the current build role for one made +// without saying (a test building the struct by hand). +func (b *natsBuilds) role() string { + if b.seat == "" { + return TheBuildMachine + } + return b.seat } func (b *natsBuilds) Close() { @@ -51,7 +69,7 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error { } publish, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - if _, err := b.js.Context().Publish(BuildWork(), body, nats.Context(publish)); err != nil { + if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil { return fmt.Errorf("cannot submit a build: %w", err) } return nil @@ -63,7 +81,7 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest, // Subscribed before the ask, so an outcome cannot arrive before there is anywhere for it to // land. Core, not the stream: the asker is waiting now, and the durable copy of this outcome is // the same event on EVENTS, which the controller records. - outcomes, err := b.js.Conn().SubscribeSync(BuildOutcome()) + outcomes, err := b.js.Conn().SubscribeSync(BuildOutcomeOf(b.role())) if err != nil { return BuildResult{}, fmt.Errorf("cannot listen for a build's outcome: %w", err) } @@ -80,7 +98,7 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest, // be assumed, because nothing else will ever say so. publish, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - if _, err := b.js.Context().Publish(BuildWork(), body, nats.Context(publish)); err != nil { + if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil { return BuildResult{}, fmt.Errorf("cannot submit a build: %w", err) } diff --git a/internal/link/builds_nats_test.go b/internal/link/builds_nats_test.go index 9bac7bd..b0feaa2 100644 --- a/internal/link/builds_nats_test.go +++ b/internal/link/builds_nats_test.go @@ -325,3 +325,39 @@ func TestNatsTwoMachinesShareTheWorkAndNeitherIsHandedMoreThanItCanTake(t *testi case <-time.After(2 * time.Second): } } + +// During the handover (ADR 0190) two build roles exist. A machine whose credential claims the retired +// one takes an ask published to that seat and answers as that seat; the asker of that seat hears it. +func TestNatsAMachineOnTheRetiredBuildRoleTakesThatRolesAsks(t *testing.T) { + js := aBusWithTheBuildRole(t) + seats := []broker.DeclaredSeat{{Name: TheBuildMachineBefore, Accepts: []string{"build"}, Emits: []string{"built"}}} + if err := broker.RaiseSeats(js, seats, map[string]broker.Holder{TheBuildMachineBefore: {Node: "anchor", Module: "builder"}}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + machine := MachineOverNATSOn(js, "anchor", TheBuildMachineBefore) + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + _ = work.Began(ctx) + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, Repository: work.Request().Repository, On: "anchor", Commit: "abc"}) + _ = work.Done() + }) + }() + + asker, err := BuildsOverNATSOn(os.Getenv("MESH_TEST_NATS"), TheBuildMachineBefore) + if err != nil { + t.Fatal(err) + } + defer asker.Close() + result, err := asker.Submit(ctx, BuildRequest{ID: "build-old-seat", Repository: "r"}, 20*time.Second) + if err != nil { + t.Fatal(err) + } + if result.On != "anchor" || result.ID != "build-old-seat" { + t.Errorf("the retired role's holder did not answer: %+v", result) + } +} diff --git a/internal/link/receive_nats.go b/internal/link/receive_nats.go index 0d812b8..a1889ee 100644 --- a/internal/link/receive_nats.go +++ b/internal/link/receive_nats.go @@ -223,10 +223,11 @@ func kindOfSubject(subject string) (string, bool) { return KindCatchUp, true case broker.ControllerFollows[3]: return KindSourceMoved, true - case BuildOutcome(): + case BuildOutcome(), BuildOutcomeOf(TheBuildMachineBefore): // A build's outcome is the role's event now, so it arrives on the events stream rather than // the control branch — and is acted on by the same handler, because what the controller does - // with it did not change (novox/hq ADR 0121). + // with it did not change (novox/hq ADR 0121). From either build role while the handover + // runs (ADR 0190): the old builder still answers on the retired seat until it is unassigned. return KindBuilt, true } return "", false