novox/mesh-controller
13
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

Rollouts advance on events, and builds and checks wait on three work queues (issue 974, design 47 milestone 2) #368

Merged
mesh-tools merged 12 commits from feat/events-and-queues into main 2026-10-17 12:52:34 +00:00
Conversation 4 Commits 12 Files Changed 49
+3828 -1492
49 changed files with 3828 additions and 1492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-builder/main.go
+26 -1
View File
@@ -27,6 +27,7 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
@@ -290,6 +291,26 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
if err := work.Began(ctx); err != nil {
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
}
// **A running check watches its pull request's key** (novox/hq ADR 0406 §4, issue 974): when the head or the trunk
// commit in it moves, the check is for a state nobody wants any more, and ends itself — nothing is asked of the forge
// (issue 726). A group's check, a try and a build watch nothing.
var superseded atomic.Bool
if w, ok := work.(interface {
WatchPull(context.Context, func(link.PullKey)) (func(), error)
}); ok && request.Check != nil {
stopWatching, err := w.WatchPull(building, func(k link.PullKey) {
superseded.Store(true)
say("superseded", fmt.Sprintf("the pull request's key moved to %.12s on %.12s while this checked %.12s on %.12s: stopped",
k.Head, k.Trunk, request.Ref, request.Check.Trunk))
cancel()
})
if err != nil {
fmt.Fprintf(os.Stderr, "the key of %s/%s#%d could not be watched, so its check runs on if the key moves: %v\n",
request.Check.Owner, request.Check.Repo, request.Check.Number, err)
} else {
defer stopWatching()
}
}
result := link.BuildResult{
ID: request.ID, Repository: request.Repository, Path: request.Path,
@@ -382,7 +403,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
if mine != nil {
killed = h.returned(mine) && err != nil
}
if killed {
if superseded.Load() && err != nil {
// Ended by its own key moving on (ADR 0406 §4): said as the check's failure, as a kill is.
result.Failed = link.SupersededByKey
say("failed", link.SupersededByKey)
} else if killed {
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
// error it ended with is the kill's consequence, not a fault of the source.
result.Failed = link.KilledByHand
cmd/mesh-controller/checks.go
+11 -3
View File
@@ -191,8 +191,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
if err := ask.Ask(ctx, request); err != nil {
return err
}
// A new head ends the check of the old one (novox/hq issue 726).
endingSupersededChecks(ctx, seat, p, request.ID, f.oneShot)
// **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4,
// issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked
// of the forge here (issue 726).
what := "its own merge-check.sh alone: " + noModuleTouched
if scope.gated() {
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
@@ -270,7 +271,7 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
Held: heldBy(ctx),
Seats: seatBases(ctx),
Source: sourceOnSeat(source),
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready,
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
Release: release, Running: runningBy(ctx)},
@@ -582,6 +583,13 @@ const maxCheckReport = 60 << 10
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
func checked(ctx context.Context, result link.BuildResult) {
// **A check stopped by its key says nothing** (novox/hq ADR 0406 §4, issue 974, review 2): it stopped because the head or
// the trunk commit moved on, and the newer request will judge. Said as an error it would name the live head — red
// statuses on the forge, a delivery refused — and written to the key it would stand in the newer request's way.
if result.Failed == link.SupersededByKey {
fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID)
return
}
sayChecked(ctx, checkedOf(result))
}
cmd/mesh-controller/checks_queue_test.go
+95
View File
@@ -0,0 +1,95 @@
package main
import (
"context"
"encoding/json"
"fmt"
"testing"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// Acceptance 5 of milestone 2 (novox/hq ADR 0406 §3, issue 974), live, with the store and a real bus: two merges into one
// repository within a minute, with two open pull requests, leave each pull request with exactly one queued check, by the
// stream's rule. Each merge moves the trunk, and mesh-delivery asks the controller to check both open pull requests again
// against it (`delivery check`, which is the controller's own path for a head announced); nothing moves a request, nothing
// asks the forge, and the queue is read back whole.
func TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
js, _ := aBuildQueue(t)
servingBus.Store(js)
t.Cleanup(func() { servingBus.Store(nil) })
holdTheDeliverySeat(t, open)
was := askDeliveryOwner
t.Cleanup(func() { askDeliveryOwner = was })
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) {
if verb != "groups" {
t.Fatalf("asked the owner %s", verb)
}
return json.RawMessage(`[]`), nil
}
// The repository the two pull requests are into, as a module the mesh builds from it.
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
heads := map[int]string{7: "7777777777777777777777777777777777777777", 8: "8888888888888888888888888888888888888888"}
check := func(number int, trunk string) {
t.Helper()
p := orderMember{ID: fmt.Sprintf("novox/mesh-controller@%.12s", heads[number]), Repository: "novox/mesh-controller",
Base: "main", Head: heads[number], Number: number, Paths: []string{"main.go"}, ModuleDirsSaid: true,
BaseSHA: trunk, Ready: number == 8}.pull()
p.MergeCheck, p.MergeCheckSaid = true, true
if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil {
t.Fatal(err)
}
}
// Announced against the trunk as it stands, then asked again after each of two merges.
for _, trunk := range []string{"1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222",
"3333333333333333333333333333333333333333"} {
check(7, trunk)
check(8, trunk)
}
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
if err != nil {
t.Fatal(err)
}
seen := map[int]string{}
for _, a := range q.Asks {
c := a.Request.Check
if c == nil {
t.Fatalf("%s is no check", a.ID)
}
if _, twice := seen[c.Number]; twice {
t.Errorf("pull request %d has two queued checks", c.Number)
}
seen[c.Number] = c.Trunk
if a.State != link.AskWaiting {
t.Errorf("%s is %s", a.ID, a.State)
}
if want := "3333333333333333333333333333333333333333"; c.Trunk != want {
t.Errorf("pull request %d is queued against %.8s; want the newest trunk %.8s", c.Number, c.Trunk, want)
}
}
if len(seen) != 2 {
t.Fatalf("queued checks for %v; want one for each of the two open pull requests", seen)
}
// By the streams' own count: the ready one on its stream, the other on its.
for stream, want := range map[string]uint64{broker.ChecksReadyStream: 1, broker.ChecksOtherStream: 1} {
info, err := js.Context().StreamInfo(stream)
if err != nil {
t.Fatal(err)
}
if info.State.Msgs != want {
t.Errorf("%s holds %d request(s); want %d", stream, info.State.Msgs, want)
}
}
}
cmd/mesh-controller/checks_test.go
+51
View File
@@ -1,9 +1,16 @@
package main
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
@@ -288,3 +295,47 @@ func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk)
}
}
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) {
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
legacy, err := js.Conn().JetStream()
if err != nil {
t.Fatal(err)
}
was := checkEvents
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
t.Cleanup(func() { checkEvents = was })
heard := make(chan *nats.Msg, 4)
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard)
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3}
checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey})
checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
select {
case m := <-heard:
var c link.Checked
_ = json.Unmarshal(m.Data, &c)
if c.ID != "broken" || c.Verdict != "error" {
t.Fatalf("the first verdict said is %+v; want only the broken check's error", c)
}
case <-time.After(5 * time.Second):
t.Fatal("nothing was said")
}
select {
case m := <-heard:
t.Fatalf("a second verdict was said: %s", m.Data)
case <-time.After(300 * time.Millisecond):
}
}
cmd/mesh-controller/delivery.go
+15 -2
View File
@@ -584,6 +584,9 @@ type orderMember struct {
// BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit
// a verdict no build judges names when the head is checked again through mesh-delivery.
BaseSHA string `json:"base_sha,omitempty"`
// Ready says the member's pull request is marked ready (novox/hq ADR 0406 §3, issue 974): its check is queued before
// every other check. Said by mesh-delivery on a ready mark.
Ready bool `json:"ready,omitempty"`
}
// pull is the member as the forge announced it.
@@ -595,7 +598,7 @@ func (m orderMember) pull() link.PullUpdated {
}
return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA,
CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed,
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid}
ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid, Ready: m.Ready}
}
// orderPair is one "before" among a group's members, why, and the rules it won over.
@@ -892,6 +895,8 @@ func deliveryCommand(ctx context.Context, args []string) error {
removed := set.String("removed", "", "the files it deletes, comma-separated")
membersJSON := set.String("members", "", "a group's members, as JSON")
group := set.String("group", "", "a delivery group's id")
ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+
"(novox/hq ADR 0406 §3)")
by := set.String("by", catalogue.DeliverySeat, "who says it")
why := set.String("why", "", "why")
without := set.String("without", "", "merges a stop leaves out, <repository>@<commit>, comma-separated: the walk's "+
@@ -907,6 +912,12 @@ func deliveryCommand(ctx context.Context, args []string) error {
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
return fmt.Errorf("the members are not readable: %w", err)
}
// Marked ready by the verb's word for every member it names, besides any member that says so itself.
if *ready {
for i := range members {
members[i].Ready = true
}
}
}
open, err := openStores(ctx)
if err != nil {
@@ -977,7 +988,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
}); err != nil {
return err
}
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository,
fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository,
short(p.Commit), *by)
return nil
case "stop":
@@ -1166,6 +1177,8 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
return "", err
}
request.Check.Group = group
// A group's check is queued as ready when any member's pull request is marked ready.
request.Check.Ready = slices.ContainsFunc(members, func(m orderMember) bool { return m.Ready })
world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "")
if err != nil {
return "", err
cmd/mesh-controller/plan_looks.go
+674
View File
@@ -0,0 +1,674 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A rollout advances on events, not on a timer (novox/hq ADR 0406 §5, issue 974). One look at the open plans runs on
// each thing that can let a plan move: a plan saved (a build's `built`, a tier's send kept, a command's move), a
// machine's report on a send, a silent key's expiry, and the controller taking the lease. The looks are asked for
// with askToLook and run one at a time by a worker, so an ask made while a look runs is not lost and many asks are
// one look. Only two things are still started by time, both named and both one-shot: a failed advance step is
// looked at again (retryLook), and the end of a walk whose end is not known is recorded when it can be
// (armWalkEnd).
// looker runs the look at the plans that events ask for.
type looker struct {
signal chan struct{} // one place: asks made together are one look
look func(ctx context.Context) (failed bool)
// gap is the least time between two looks, so a plan that kept changing could not make the worker spin.
gap time.Duration
// first runs before the first look: what a controller that took the lease sets up.
first func(ctx context.Context)
mu sync.Mutex
retry *time.Timer
backoff time.Duration
base time.Duration
most time.Duration
}
const (
retryLookFirst = 5 * time.Second
retryLookMost = time.Minute
lookGap = 200 * time.Millisecond
)
func newLooker(look func(ctx context.Context) bool) *looker {
return &looker{signal: make(chan struct{}, 1), look: look, gap: lookGap, base: retryLookFirst, most: retryLookMost}
}
// ask has a look run, soon; never blocks.
func (l *looker) ask() {
select {
case l.signal <- struct{}{}:
default:
}
}
// run takes the asks one at a time until the context ends.
func (l *looker) run(ctx context.Context) {
if l.first != nil {
l.first(ctx)
}
for {
select {
case <-ctx.Done():
l.mu.Lock()
if l.retry != nil {
l.retry.Stop()
}
l.mu.Unlock()
return
case <-l.signal:
}
l.after(l.look(ctx))
if l.gap > 0 {
select {
case <-ctx.Done():
case <-time.After(l.gap):
}
}
}
}
// after takes the outcome of a look: a failed one is looked at again later, a good one forgets the failures.
func (l *looker) after(failed bool) {
l.mu.Lock()
defer l.mu.Unlock()
if !failed {
l.backoff = 0
if l.retry != nil {
l.retry.Stop()
l.retry = nil
}
return
}
if l.retry != nil {
return // one is armed already
}
switch {
case l.backoff == 0:
l.backoff = l.base
case l.backoff*2 > l.most:
l.backoff = l.most
default:
l.backoff *= 2
}
// retries: an advance step that failed (store, bus or send). One at a time, five seconds doubling to a minute, forgotten by
// the first look that fails nothing; the step is otherwise started by an event, and a failure is none.
l.retry = time.AfterFunc(l.backoff, func() {
l.mu.Lock()
l.retry = nil
l.mu.Unlock()
l.ask()
})
}
// theLooker is the serving controller's; nil in a command, whose looks are its own.
var (
lookerMu sync.RWMutex
theLooker *looker
)
// askToLook has the serving controller look at the open plans soon: a plan was saved, a machine reported on a send,
// a silent key expired. Nothing in a process that is not serving.
func askToLook() {
lookerMu.RLock()
l := theLooker
lookerMu.RUnlock()
if l != nil {
l.ask()
}
}
// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
func startPlanLooks(ctx context.Context, open *stores) {
l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) })
l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) }
lookerMu.Lock()
theLooker = l
lookerMu.Unlock()
go func() {
<-ctx.Done()
lookerMu.Lock()
if theLooker == l {
theLooker = nil
}
lookerMu.Unlock()
}()
go l.run(ctx)
l.ask()
}
// applySilentAfter is how long a machine may say nothing on a send before the plan names it silent: the mesh's
// ApplySilentAfter, a variable so a test of the wait needs no quarter of an hour.
var applySilentAfter = inventory.ApplySilentAfter
// startRolloutEvents starts what a rollout advances on, for as long as the context lives (novox/hq ADR 0406 §5, issue 974):
// the wait for each machine's report as a key on the bus that expires (where the bus will not take it a timer of its own
// for that wait looks instead), a provisioner client's words, and the look at the plans, asked for first so a plan left
// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one.
func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) {
if err := startRolloutSilent(ctx, js); err != nil {
fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err)
}
if err := startProvAppliedLooks(ctx, js); err != nil {
fmt.Printf("plans: the provisioner clients' words could not be watched (%v); a walk waits for them until the next look\n", err)
}
if err := startPlanMovedLooks(ctx, js); err != nil {
fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err)
}
startPlanLooks(ctx, open)
}
// servingPlanSaved is what the serving controller does for each plan it keeps: said as plan-moved, looked at (a build, a
// send kept or a command's move is followed by the look that asks the next tier, in place of the timer that did), and,
// for a walk that ended, its merges' check builds forgotten (novox/hq ADR 0350 §5).
func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inventory.Inventory, inventory.Plan) {
return func(_ *inventory.Inventory, p inventory.Plan) {
sayPlanMoved(ctx, bus, open.inventory, p)
planKept(p)
if p.WalkSaid == inventory.WalkEnded {
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
}
}
}
// planLookPasses is the most times one look goes over the plans again because the first pass changed which are open.
const planLookPasses = 6
// lookAtPlans moves every open plan as far as the facts allow, as the ticker did each pass: what waits for a gate is
// released, the plans advanced, the wait for each machine's report kept as a key on the bus, and the phases of the
// walks that ended recorded. It says whether a step failed. Waits for the plans rather than giving up: an ask is
// answered once.
func lookAtPlans(ctx context.Context, open *stores) (failed bool) {
release, err := open.inventory.HoldPlans(ctx, true)
if err != nil {
fmt.Printf("plans: cannot hold them: %v\n", err)
return true
}
func() {
defer release()
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
if _, err := releaseBacklog(ctx, open, ""); err != nil {
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
failed = true
}
// Again while the plans that are open changed: a plan that ended lets a deferred one before it start, and a
// batch cut makes one.
for pass := 0; pass < planLookPasses; pass++ {
before := openPlansNow(ctx, open.inventory)
if advanceHeld(ctx, open) {
failed = true
}
if openPlansNow(ctx, open.inventory) == before {
break
}
}
if err := syncSilentKeys(ctx, open); err != nil {
fmt.Printf("plans: the wait for the machines' reports could not be kept on the bus: %v\n", err)
failed = true // so the look is retried
}
}()
keepWalkPhases(ctx, open)
return failed
}
// advancePlans is one pass over the open plans taking the plans without waiting for them, for a caller that wants a
// single step rather than the look (the tests of the steps).
func advancePlans(ctx context.Context, open *stores) {
release, err := open.inventory.HoldPlans(ctx, false)
if err != nil {
if !errors.Is(err, inventory.ErrPlansBusy) {
fmt.Printf("plans: cannot hold them: %v\n", err)
}
return
}
defer release()
if _, err := releaseBacklog(ctx, open, ""); err != nil {
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
}
advanceHeld(ctx, open)
}
// openPlansNow is which plans are open and in what state, for telling a pass that changed them from one that did not.
func openPlansNow(ctx context.Context, inv *inventory.Inventory) string {
plans, err := inv.OpenPlans(ctx)
if err != nil {
return "unreadable " + time.Now().String()
}
ids := make([]string, 0, len(plans))
for _, p := range plans {
ids = append(ids, p.ID+":"+string(p.State))
}
sort.Strings(ids)
return strings.Join(ids, ",")
}
// --- the wait for a machine's report, as keys on the bus ---
var (
silentMu sync.RWMutex
silentKeyed silentClock // nil in a process that is not the serving controller
)
// setSilentClock makes clock the rollout's; nil takes it away.
func setSilentClock(c silentClock) {
silentMu.Lock()
defer silentMu.Unlock()
silentKeyed = c
}
func silentClockOf() silentClock {
silentMu.RLock()
defer silentMu.RUnlock()
return silentKeyed
}
// silentWaitsOf is every machine an open plan waits on for a report, with when it was sent: the machines a walk sent
// that have not reported on the send and are not silent yet (walkReports), the machine a gate of a first machine
// judges, and the machines a release's gates wait on. Pure.
func silentWaitsOf(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[silentWait]time.Time {
waits := map[silentWait]time.Time{}
put := func(w silentWait, at time.Time) {
if at.IsZero() {
return
}
if was, ok := waits[w]; !ok || at.Before(was) {
waits[w] = at
}
}
if !p.Open() {
return waits
}
if r := p.Release; r != nil {
gates := []*inventory.PlanGate{r.Gate}
for _, g := range r.Gates {
gates = append(gates, g)
}
for _, g := range gates {
if g == nil || g.Verdict != "" || g.Silent || g.Pending || g.Since == nil {
continue
}
for _, node := range g.Machines {
put(silentWait{Plan: p.ID, Tier: 0, Node: node}, *g.Since)
}
}
return waits
}
if p.Waiting() {
return waits
}
tierOf := func(module string) int {
for i, tier := range p.Tiers {
if slices.Contains(tier, module) {
return i
}
}
return 0
}
waiting, _, _ := walkReports(p, reports, now, prov)
for module, s := range p.Modules {
if s == nil {
continue
}
for _, node := range waiting {
sent, ok := s.Rest[node]
if !ok {
continue
}
at := sent.At
if at.IsZero() && s.SentAt != nil {
at = *s.SentAt
}
put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, at)
}
if g := s.Gate; g != nil && g.Verdict == "" && !g.Silent && g.Since != nil && s.FirstAt != nil {
for _, node := range g.Machines {
put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, *g.Since)
}
}
}
return waits
}
// silentTTL is how long a wait's key lives: what is left of ApplySilentAfter from the send, and a second over, so the
// look it wakes finds the machine silent and not a fraction early.
func silentTTL(sent, now time.Time) time.Duration {
left := sent.Add(applySilentAfter).Sub(now) + time.Second
if left < time.Second {
return time.Second
}
return left
}
// silentDiff is what the keys on the bus need to be as the waits are: the waits to put a key for (none yet, or the
// key is of an earlier send) and the keys to delete (the wait is over). Pure.
func silentDiff(want, held map[silentWait]time.Time) (arm, drop []silentWait) {
for w, sent := range want {
if was, ok := held[w]; !ok || !was.Equal(sent) {
arm = append(arm, w)
}
}
for w := range held {
if _, ok := want[w]; !ok {
drop = append(drop, w)
}
}
return arm, drop
}
// syncSilentKeys keeps the keys on the bus as the open plans' waits are: a wait without a key gets one that lives out
// its time, a send made again gets its key again, and a key whose wait is over — the machine reported on the send, the
// gate has its verdict, the plan ended — is deleted. A report deletes the key by way of the look it asks for (the
// look alone knows the report was on that send). Only the lease holder looks, so only it writes.
func syncSilentKeys(ctx context.Context, open *stores) error {
c := silentClockOf()
inv := open.inventory
plans, err := inv.OpenPlans(ctx)
if err != nil {
return err
}
reports, err := inv.LastReports(ctx)
if err != nil {
return err
}
now := time.Now().UTC()
want := map[silentWait]time.Time{}
for _, p := range plans {
for w, at := range silentWaitsOf(p, reports, now, provLookup(ctx)) {
want[w] = at
}
}
return keepSilentKeys(ctx, c, want, now)
}
// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys
// cannot be read, one cannot be put — the wait is looked at by a timer of its own, once, and the error is given back so
// the look is retried.
func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error {
byTimer := func() {
for w, sent := range want {
silentByTimer(w, sent, silentTTL(sent, now))
}
}
if c == nil {
byTimer()
return nil
}
held, err := c.Held(ctx)
if err != nil {
byTimer()
return err
}
arm, drop := silentDiff(want, held)
var first error
for _, w := range arm {
if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
silentByTimer(w, want[w], silentTTL(want[w], now))
if first == nil {
first = err
}
}
}
for _, w := range drop {
if err := c.Reported(ctx, w); err != nil && first == nil {
first = err
}
}
return first
}
var silentTimers = struct {
mu sync.Mutex
timers map[silentWait]time.Time
}{timers: map[silentWait]time.Time{}}
// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key
// (DegradableGrants says why that can be): never silently never. Once for each send.
func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) {
silentTimers.mu.Lock()
defer silentTimers.mu.Unlock()
if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) {
return
}
silentTimers.timers[w] = sent
// retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once.
time.AfterFunc(ttl, func() {
silentTimers.mu.Lock()
if silentTimers.timers[w].Equal(sent) {
delete(silentTimers.timers, w)
}
silentTimers.mu.Unlock()
askToLook()
})
}
// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
// controller is the lease holder here, the only one that writes it. The watcher asks for a look when a key expires.
func startRolloutSilent(ctx context.Context, js *broker.JetStream) error {
api, err := jetstream.New(js.Conn())
if err != nil {
return err
}
clock, err := newBusSilent(ctx, api, func(silentWait) { askToLook() })
if err != nil {
return err
}
setSilentClock(clock)
go func() {
<-ctx.Done()
silentMu.Lock()
if silentKeyed == silentClock(clock) {
silentKeyed = nil
}
silentMu.Unlock()
}()
return nil
}
// startProvAppliedLooks has the plans looked at when a machine's provisioner client says a word (`prov.applied.<node>.<module>`):
// a send that handed modules to the provisioner is applied only once the client said so too, and without this the walk
// would wait for its silent key (ADR 0362, ADR 0406 §5, issue 974). New words only: the first look reads the rest.
func startProvAppliedLooks(ctx context.Context, js *broker.JetStream) error {
api, err := jetstream.New(js.Conn())
if err != nil {
return err
}
stream, err := api.Stream(ctx, broker.ProvAppliedStream)
if err != nil {
return fmt.Errorf("the provisioner clients' words: %w", err)
}
consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverNewPolicy})
if err != nil {
return fmt.Errorf("watching the provisioner clients' words: %w", err)
}
consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() })
if err != nil {
return fmt.Errorf("watching the provisioner clients' words: %w", err)
}
go func() {
<-ctx.Done()
consuming.Stop()
}()
return nil
}
// startPlanMovedLooks has the plans looked at when any process says one moved: the serving controller follows its own
// `plan-moved`, so a plan a person moved with a verb at the control node's terminal — a process this one did not spawn,
// whose saves it does not hear — is looked at all the same (novox/hq ADR 0406 §2, §5, issue 974). New messages only; the
// first look reads the rest. A look its own save asks is the same look: asks are one.
func startPlanMovedLooks(ctx context.Context, js *broker.JetStream) error {
api, err := jetstream.New(js.Conn())
if err != nil {
return err
}
stream, err := api.Stream(ctx, broker.EventsStream)
if err != nil {
return fmt.Errorf("the events stream: %w", err)
}
consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{
FilterSubjects: []string{link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved)},
DeliverPolicy: jetstream.DeliverNewPolicy})
if err != nil {
return fmt.Errorf("following the plans moved: %w", err)
}
consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() })
if err != nil {
return fmt.Errorf("following the plans moved: %w", err)
}
go func() {
<-ctx.Done()
consuming.Stop()
}()
return nil
}
// --- the end of a walk whose end is not known ---
var walkEnds = struct {
mu sync.Mutex
timers map[string]*time.Timer
}{timers: map[string]*time.Timer{}}
// armWalkEnd has the walk recorded once its end can be known: a walk that ended with a machine silent is recorded
// without its total ApplySilentAfter and a minute after its last save (recordWalkPhases), which no event says. Armed at
// each save of the done walk, so it counts from the last.
func armWalkEnd(id string, savedAt time.Time) {
d := time.Until(savedAt.Add(inventory.ApplySilentAfter + time.Minute + time.Second))
if d <= 0 {
return
}
walkEnds.mu.Lock()
defer walkEnds.mu.Unlock()
if t := walkEnds.timers[id]; t != nil {
t.Stop()
}
// retries: recording the end of a walk whose end is not known. One per walk, once, from its last save; the first
// look after the lease was taken arms it again for the walks still unrecorded.
walkEnds.timers[id] = time.AfterFunc(d, func() {
walkEnds.mu.Lock()
delete(walkEnds.timers, id)
walkEnds.mu.Unlock()
askToLook()
})
}
// rearmWalkEnds is armWalkEnd for the walks that ended lately, for a controller that has just taken the lease.
func rearmWalkEnds(ctx context.Context, open *stores) {
recent, err := open.inventory.RecentPlans(ctx, 30)
if err != nil {
fmt.Printf("plans: the walks ended lately could not be read to record their phases: %v\n", err)
return
}
for _, p := range recent {
if p.State == inventory.PlanDone && p.Release == nil && time.Since(p.Updated) <= 2*time.Hour {
armWalkEnd(p.ID, p.Updated)
}
}
}
// planKept is what the serving controller does when a plan was saved: a look is asked for, so a send, a built tier or
// a command's move is followed by the plan's next step; and a walk that ended is armed to be recorded.
func planKept(p inventory.Plan) {
if p.State == inventory.PlanDone && p.Release == nil {
armWalkEnd(p.ID, time.Now())
}
askToLook()
}
// --- `sent`: a tier sent to a machine, said ---
// sendMade is one machine a plan's tier was sent to.
type sendMade struct {
Tier int
Node string
Modules []string
At time.Time
}
// sendsMade collects the sends one step of a plan made, to be said once the plan is saved.
type sendsMade struct {
mu sync.Mutex
list []sendMade
}
type sendsMadeKey struct{}
func withSendsMade(ctx context.Context, s *sendsMade) context.Context {
return context.WithValue(ctx, sendsMadeKey{}, s)
}
// noteSends records that modules of the plan's tier were sent to the nodes.
func noteSends(ctx context.Context, tier int, nodes, modules []string, at time.Time) {
s, _ := ctx.Value(sendsMadeKey{}).(*sendsMade)
if s == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
for _, n := range nodes {
s.list = append(s.list, sendMade{Tier: tier, Node: n, Modules: slices.Clone(modules), At: at})
}
}
// planSentBody is the body of the seat event `sent` (novox/hq ADR 0406 §5, issue 974): the plan, the tier, the machine
// and the modules it was sent, and when.
type planSentBody struct {
Event string `json:"event"`
Plan string `json:"plan"`
Tier int `json:"tier"`
Node string `json:"node"`
Modules []string `json:"modules"`
At time.Time `json:"at"`
}
// sentBodies are the events a step's sends are said as.
func sentBodies(plan string, sends *sendsMade) []planSentBody {
sends.mu.Lock()
defer sends.mu.Unlock()
out := make([]planSentBody, 0, len(sends.list))
for _, s := range sends.list {
out = append(out, planSentBody{Event: link.KeySent, Plan: plan, Tier: s.Tier, Node: s.Node,
Modules: s.Modules, At: s.At})
}
return out
}
// saySends says each send a step made as `sent`, once the plan that records it is saved. What an outsider hears; the
// controller's own next step does not wait for it.
func saySends(ctx context.Context, p inventory.Plan, sends *sendsMade) {
bus := checkEvents
if bus == nil {
return
}
for _, body := range sentBodies(p.ID, sends) {
raw, err := json.Marshal(body)
if err != nil {
continue
}
go func() {
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer stop()
if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySent, raw); err != nil {
fmt.Printf("%s: tier %d was sent to %s, and could not be said as sent: %v\n", body.Plan, body.Tier, body.Node, err)
}
}()
}
}
cmd/mesh-controller/plan_looks_test.go
+351
View File
@@ -0,0 +1,351 @@
package main
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) {
sent := time.Now().Add(-sentAgo).UTC()
p := inventory.Plan{ID: "plan-9", State: inventory.PlanRolling, Tiers: [][]string{{"a", "b"}}, Tier: 0,
Modules: map[string]*inventory.PlanModule{
"a": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
"b": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
}}
r := inventory.Reported{Node: "ace", Sent: &sent}
if reported {
at := sent.Add(time.Minute)
r.At, r.Current, r.Declared, r.Outcome = &at, true, "d1", inventory.OutcomeApplied
}
return p, []inventory.Reported{r}
}
// A machine a plan sent and heard nothing from is a wait with a key, from when it was sent (issue 974, novox/hq ADR
// 0406 §5); one that reported on the send, or that is silent already, is none.
func TestAPlanWaitsOnTheMachinesItSentAndHeardNothingFrom(t *testing.T) {
now := time.Now().UTC()
p, reports := rolloutOf(5*time.Minute, false)
waits := silentWaitsOf(p, reports, now, nil)
if len(waits) != 1 {
t.Fatalf("waits = %v; want the one machine", waits)
}
for w, at := range waits {
if w.Plan != "plan-9" || w.Node != "ace" || w.Tier != 0 || now.Sub(at) < 4*time.Minute || now.Sub(at) > 6*time.Minute {
t.Errorf("wait %+v sent at %v is wrong", w, at)
}
}
p, reports = rolloutOf(5*time.Minute, true)
if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
t.Errorf("a machine that reported on its send is waited on: %v", waits)
}
p, reports = rolloutOf(inventory.ApplySilentAfter+time.Minute, false)
if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
t.Errorf("a machine silent past its time is waited on again: %v", waits)
}
p, reports = rolloutOf(time.Minute, false)
p.State = inventory.PlanDone
if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
t.Errorf("a plan that ended is waiting: %v", waits)
}
}
// A release's open gate waits on its machines from when it began judging, and its verdict ends the wait.
func TestAReleasesOpenGateWaitsOnItsMachines(t *testing.T) {
since := time.Now().Add(-time.Minute).UTC()
g := &inventory.PlanGate{Machines: []string{"g14", "novox"}, Since: &since}
p := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: g}}
waits := silentWaitsOf(p, nil, time.Now(), nil)
if len(waits) != 2 {
t.Fatalf("waits = %v; want g14 and novox", waits)
}
g.Verdict = inventory.GatePassed
if waits := silentWaitsOf(p, nil, time.Now(), nil); len(waits) != 0 {
t.Errorf("a gate with its verdict still waits: %v", waits)
}
}
// The keys follow the waits: a wait with no key, or a key of an earlier send, is armed; a key with no wait is deleted.
func TestTheKeysFollowTheWaits(t *testing.T) {
t1 := time.Now().UTC()
t2 := t1.Add(time.Minute)
a, b, c := silentWait{"p", 0, "a"}, silentWait{"p", 0, "b"}, silentWait{"p", 0, "c"}
arm, drop := silentDiff(map[silentWait]time.Time{a: t1, b: t2, c: t1}, map[silentWait]time.Time{a: t1, b: t1, silentWait{"p", 1, "a"}: t1})
if len(arm) != 2 || !(arm[0] == b || arm[1] == b) || !(arm[0] == c || arm[1] == c) {
t.Errorf("armed %v; want b (sent again) and c (no key)", arm)
}
if len(drop) != 1 || drop[0] != (silentWait{"p", 1, "a"}) {
t.Errorf("dropped %v; want the key with no wait", drop)
}
if ttl := silentTTL(t1, t1.Add(inventory.ApplySilentAfter+time.Hour)); ttl != time.Second {
t.Errorf("a wait long over has a key of %v; want the least, one second", ttl)
}
}
func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan struct{}) *looker {
l := newLooker(func(ctx context.Context) bool {
looks.Add(1)
if block != nil {
<-block
}
return failures.Add(-1) >= 0
})
l.gap, l.base, l.most = 0, 20*time.Millisecond, 80*time.Millisecond
return l
}
func eventually(t *testing.T, what string, ok func() bool) {
t.Helper()
for end := time.Now().Add(15 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
if ok() {
return
}
}
t.Fatalf("never: %s", what)
}
// Asks made while a look runs are not lost and are one look after it, not one each.
func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) {
var failures, looks atomic.Int32
block := make(chan struct{})
l := lookerCounting(&failures, &looks, block)
ctx, stop := context.WithCancel(context.Background())
defer stop()
go l.run(ctx)
l.ask()
eventually(t, "the first look starts", func() bool { return looks.Load() == 1 })
for i := 0; i < 20; i++ {
l.ask()
}
block <- struct{}{} // the first look ends
eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 })
block <- struct{}{}
time.Sleep(60 * time.Millisecond)
if n := looks.Load(); n != 2 {
t.Errorf("%d looks for 21 asks; want 2", n)
}
}
// A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails
// nothing ends the retries (novox/hq ADR 0406 §5; the one named retry).
func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) {
var failures, looks atomic.Int32
failures.Store(3) // three looks fail, then one is good
l := lookerCounting(&failures, &looks, nil)
ctx, stop := context.WithCancel(context.Background())
defer stop()
go l.run(ctx)
l.ask()
eventually(t, "four looks", func() bool { return looks.Load() == 4 })
time.Sleep(200 * time.Millisecond)
if n := looks.Load(); n != 4 {
t.Errorf("%d looks; the good one should have ended the retries at 4", n)
}
l.mu.Lock()
armed, backoff := l.retry != nil, l.backoff
l.mu.Unlock()
if armed || backoff != 0 {
t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff)
}
}
func TestTheRetryWaitDoublesToItsMost(t *testing.T) {
l := newLooker(func(context.Context) bool { return true })
l.base, l.most = 5*time.Second, time.Minute
var seen []time.Duration
for i := 0; i < 6; i++ {
l.after(true)
seen = append(seen, l.backoff)
l.retry.Stop()
l.retry = nil
}
want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute, time.Minute}
for i := range want {
if seen[i] != want[i] {
t.Fatalf("waits %v; want %v", seen, want)
}
}
}
// A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known.
func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) {
var failures, looks atomic.Int32
l := lookerCounting(&failures, &looks, nil)
lookerMu.Lock()
theLooker = l
lookerMu.Unlock()
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
ctx, stop := context.WithCancel(context.Background())
defer stop()
go l.run(ctx)
planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone})
t.Cleanup(func() {
walkEnds.mu.Lock()
defer walkEnds.mu.Unlock()
if t := walkEnds.timers["walk-1"]; t != nil {
t.Stop()
}
delete(walkEnds.timers, "walk-1")
})
eventually(t, "a look", func() bool { return looks.Load() == 1 })
walkEnds.mu.Lock()
_, armed := walkEnds.timers["walk-1"]
walkEnds.mu.Unlock()
if !armed {
t.Error("a walk saved done is not armed to be recorded")
}
askToLook() // no panic, no looker, once it is taken away
}
// The tier-sends a step made are said once each, per machine, with the modules it was sent.
func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) {
sends := &sendsMade{}
ctx := withSendsMade(context.Background(), sends)
at := time.Now().UTC()
noteSends(ctx, 2, []string{"ace"}, []string{"a", "b"}, at)
noteSends(ctx, 2, []string{"g14"}, []string{"a"}, at)
noteSends(context.Background(), 2, []string{"nobody"}, []string{"a"}, at) // outside a step: nothing kept
got := sentBodies("plan-9", sends)
if len(got) != 2 || got[0].Node != "ace" || len(got[0].Modules) != 2 || got[0].Tier != 2 || got[0].Plan != "plan-9" ||
got[0].Event != "sent" || !got[0].At.Equal(at) || got[1].Node != "g14" {
t.Errorf("sent bodies = %+v", got)
}
}
// A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5).
func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) {
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
var failures, looks atomic.Int32
l := lookerCounting(&failures, &looks, nil)
lookerMu.Lock()
theLooker = l
lookerMu.Unlock()
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) })
ctx, stop := context.WithCancel(context.Background())
defer stop()
go l.run(ctx)
if err := startRolloutSilent(ctx, js); err != nil {
t.Fatal(err)
}
c := silentClockOf()
w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"}
if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil {
t.Fatal(err)
}
if err := c.Reported(ctx, w); err != nil {
t.Fatal(err)
}
time.Sleep(1800 * time.Millisecond)
if n := looks.Load(); n != 0 {
t.Fatalf("a key a report deleted asked for %d looks", n)
}
if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil {
t.Fatal(err)
}
eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 })
}
// A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only
// once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said.
func TestAProvisionerClientsWordAsksForALook(t *testing.T) {
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
var failures, looks atomic.Int32
l := lookerCounting(&failures, &looks, nil)
lookerMu.Lock()
theLooker = l
lookerMu.Unlock()
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
ctx, stop := context.WithCancel(context.Background())
defer stop()
go l.run(ctx)
if err := startProvAppliedLooks(ctx, js); err != nil {
t.Fatal(err)
}
legacy, err := js.Conn().JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil {
t.Fatal(err)
}
eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 })
}
type brokenClock struct{ silentClock }
func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
return nil, errors.New("the bus is down")
}
// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own
// and the error is given back, so the look is marked failed and retried (issue 974, review 1).
func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
sent := time.Now().UTC()
t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent)
if err == nil {
t.Fatal("no error given back")
}
silentTimers.mu.Lock()
armed := silentTimers.timers[w].Equal(sent)
silentTimers.mu.Unlock()
if !armed {
t.Error("no timer of its own watches the wait")
}
}
// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
// looked at: the controller follows its own plan-moved (issue 974, round 1).
func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) {
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
var failures, looks atomic.Int32
l := lookerCounting(&failures, &looks, nil)
lookerMu.Lock()
theLooker = l
lookerMu.Unlock()
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
ctx, stop := context.WithCancel(context.Background())
defer stop()
go l.run(ctx)
if err := startPlanMovedLooks(ctx, js); err != nil {
t.Fatal(err)
}
legacy, err := js.Conn().JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil {
t.Fatal(err)
}
eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 })
}
cmd/mesh-controller/plan_moved_every_writer_test.go
+140
View File
@@ -0,0 +1,140 @@
package main
import (
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there
// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that
// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed.
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
var found []string
for _, root := range []string{"../../cmd", "../../internal"} {
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return nil
}
raw, err := os.ReadFile(path)
if err != nil {
return err
}
text := string(raw)
if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
found = append(found, path+" writes the plan table")
}
if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
found = append(found, path+" calls writePlan")
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
if len(found) > 0 {
t.Fatalf("a plan is written without plan-moved being said:\n%s", strings.Join(found, "\n"))
}
raw, err := os.ReadFile("../../internal/inventory/plans.go")
if err != nil {
t.Fatal(err)
}
text := string(raw)
if n := len(write.FindAllString(text, -1)); n != 1 {
t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n)
}
calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1)
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
next := strings.Index(text[save:], "\nfunc ")
if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next {
t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls))
}
if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") {
t.Error("SavePlansMoving does not say each plan it wrote")
}
}
// movesSaid records what the hook hears: for each plan, the revisions it was said at.
type movesSaid struct {
mu sync.Mutex
revs map[string][]int64
}
func (s *movesSaid) hear(_ *inventory.Inventory, p inventory.Plan) {
s.mu.Lock()
defer s.mu.Unlock()
s.revs[p.ID] = append(s.revs[p.ID], p.Revision)
}
// sayingEverySave installs a PlanSaved that records, as the serving controller's says plan-moved.
func sayingEverySave(t *testing.T) *movesSaid {
t.Helper()
s := &movesSaid{revs: map[string][]int64{}}
was := inventory.PlanSaved
inventory.PlanSaved = s.hear
t.Cleanup(func() { inventory.PlanSaved = was })
return s
}
// noUnsaid fails the test for a plan whose stored revision is not one the hook heard: a move nobody said.
func (s *movesSaid) noUnsaid(t *testing.T, open *stores) {
t.Helper()
recent, err := open.inventory.RecentPlans(t.Context(), 50)
if err != nil {
t.Fatal(err)
}
s.mu.Lock()
defer s.mu.Unlock()
if len(recent) == 0 {
t.Fatal("no plan was kept")
}
for _, p := range recent {
revs := s.revs[p.ID]
if len(revs) == 0 || revs[len(revs)-1] != p.Revision {
t.Errorf("%s is kept at revision %d and was said at %v: a move nobody said", p.ID, p.Revision, revs)
}
}
}
// Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that
// is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2).
func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksWithPaths(t)
heard := sayingEverySave(t)
hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second))
if _, bs := walks(t, open); len(bs) != 1 {
t.Fatalf("%d batch(es) opened", len(bs))
}
heard.noUnsaid(t, open)
cutAt(t, open, t0.Add(91*time.Second))
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs))
}
heard.noUnsaid(t, open)
counted := 0
for _, r := range heard.revs {
counted += len(r)
}
cutAt(t, open, t0.Add(92*time.Second))
again := 0
for _, r := range heard.revs {
again += len(r)
}
if again != counted {
t.Errorf("a look that changed nothing said %d more move(s)", again-counted)
}
}
cmd/mesh-controller/push.go
+5 -10
View File
@@ -180,9 +180,10 @@ func serve(ctx context.Context) (err error) {
// And build results nobody was waiting for. A build triggered any other way than `build`
// would otherwise be reported into the void, which is the same as not reporting it.
server.Records(builds{inv, open})
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
// machines to report moves when they have, and a plan left by a replaced controller resumes.
go planTicker(ctx, open)
// Open plans move on events (novox/hq ADR 0406 §5, issue 974): a tier is asked on the `built` and the send of the
// tier before it, a walk ends on its machines' reports, a silent machine is a key that expired, and a plan left
// by a replaced controller resumes from the first look, which this asks for. No timer looks every so often.
startRolloutEvents(ctx, open, server.JetStream())
// And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes,
// which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it;
// nothing polls (issue 973).
@@ -252,13 +253,7 @@ func serve(ctx context.Context) (err error) {
// (novox/hq to-be 45 §9).
checkEvents = bus
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) {
sayPlanMoved(ctx, bus, open.inventory, p)
// A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5).
if p.WalkSaid == inventory.WalkEnded {
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
}
}
inventory.PlanSaved = servingPlanSaved(ctx, bus, open)
if err := server.Checks(following{open: open}); err != nil {
return err
}
cmd/mesh-controller/queue.go
+2 -35
View File
@@ -80,30 +80,6 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
return nil
}
// keepingTheQueueInOrder runs the build queue's passes while this controller acts (novox/hq issue 558):
// the one process that puts the queue in order, so no two passes pause and resume it over each other.
// The build seat is read from the catalogue at most every half minute.
func keepingTheQueueInOrder(ctx context.Context, open *stores, js *broker.JetStream) {
if js == nil {
return
}
seat, readAt := link.TheBuildMachine, time.Time{}
seatNow := func() string {
if time.Since(readAt) < watchEvery {
return seat
}
if entries, err := open.inventory.Catalogued(ctx); err == nil {
seat, readAt = buildSeatAmong(entries), time.Now()
} else {
fmt.Printf("the build seat is read as %s: the catalogue cannot be read: %v\n", seat, err)
}
return seat
}
link.KeepingInOrder(ctx, js, seatNow, link.Holding, func(format string, args ...any) {
fmt.Printf(format+"\n", args...)
})
}
// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
@@ -303,16 +279,8 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
}
}
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
// **Where it waits now** (novox/hq issue 558): moved into order since it was read, a waiting ask has a
// later sequence, and the worker handing out what was behind its old one is not it being taken.
if ask.State == link.AskWaiting {
if q, err := link.ReadQueue(ctx, js, seat); err == nil {
if now, ok := q.Find(ask.ID); ok && now.State == link.AskWaiting {
ask.Seq = now.Seq
}
}
}
// The worker of the queue it waits in: the seat's own, or one of the two check queues' (novox/hq ADR 0406 §3).
worker := link.WorkerOf(seat, ask)
switch ask.State {
case link.AskWaiting:
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
@@ -334,7 +302,6 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
"ends it there", ask.ID, started, ask.ID)
}
}
// Found by its id when it was moved into order since it was read (novox/hq issue 558).
removed, err := link.DeleteAsk(ctx, js, seat, ask)
if err != nil {
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
cmd/mesh-controller/release.go
+2 -2
View File
@@ -1023,12 +1023,12 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx))
}
if !reported {
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
if now.Sub(*g.Since) > applySilentAfter {
g.Silent = true
r.Silent = append(r.Silent, node)
changed = true
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
inventory.ApplySilentAfter))
applySilentAfter))
continue
}
judging = append(judging, node+": not yet reported on its send")
cmd/mesh-controller/release_plan.go
+35 -53
View File
@@ -24,9 +24,9 @@ import (
//
// The handler that hears the merge computes the plan from the catalogue's one dependency relation,
// writes it to the store, asks the first tier and returns — the receive loop is never held by a
// build. Every outcome taken in advances the plan it belongs to; a ticker advances what outcomes
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
// plan where it left it.
// build. Every outcome taken in advances the plan it belongs to; a look at the plans, asked for by each event that can
// move one (plan_looks.go, ADR 0406 §5), advances what outcomes alone cannot (a tier waiting for machines to report);
// a controller replaced mid-plan finds the plan where it left it and looks first.
// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since
// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on.
@@ -668,29 +668,6 @@ func askedAny(p inventory.Plan) bool {
return false
}
// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
//
// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two
// controllers — the old and the new while a machine hands its controller over — would each ask a
// tier the other had just asked. Taken without waiting: whoever holds the plans is moving them.
func advancePlans(ctx context.Context, open *stores) {
release, err := open.inventory.HoldPlans(ctx, false)
if err != nil {
if !errors.Is(err, inventory.ErrPlansBusy) {
fmt.Printf("plans: cannot hold them: %v\n", err)
}
return
}
defer release()
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
if _, err := releaseBacklog(ctx, open, ""); err != nil {
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
}
advanceHeld(ctx, open)
}
// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold
// on the plans so it never lengthens it: measured, never acted on, and an error only said.
func keepWalkPhases(ctx context.Context, open *stores) {
@@ -699,8 +676,13 @@ func keepWalkPhases(ctx context.Context, open *stores) {
}
}
// advanceHeld is advancePlans for a caller already holding the plans.
func advanceHeld(ctx context.Context, open *stores) {
// advanceHeld moves every open plan as far as the facts allow, for a caller holding the plans: a tier whose modules
// are all built and whose gates are applied gives way to the next; the last tier done is the plan done. It says
// whether a step failed, which the look at the plans (lookAtPlans) retries.
//
// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two controllers — the
// old and the new while a machine hands its controller over — would each ask a tier the other had just asked.
func advanceHeld(ctx context.Context, open *stores) (failed bool) {
inv := open.inventory
// A walk that ended lets the next batch be cut at once, not at a window's next expiry (novox/hq ADR 0276, ADR 0407).
defer func() {
@@ -711,15 +693,15 @@ func advanceHeld(ctx context.Context, open *stores) {
plans, err := inv.OpenPlans(ctx)
if err != nil {
fmt.Printf("plans: cannot read them: %v\n", err)
return
return true
}
if len(plans) == 0 {
return
return false
}
edges, err := inv.Dependencies(ctx)
if err != nil {
fmt.Printf("plans: cannot read the dependencies: %v\n", err)
return
return true
}
rollsOut := func(module string) bool {
u, err := inv.UpgradeOf(ctx, module)
@@ -733,10 +715,12 @@ func advanceHeld(ctx context.Context, open *stores) {
// stood still for half an hour was written every few seconds, its revision moved, `plan-moved`
// said on the bus, and its last save read as when it began building.
before := planSnapshot(*p)
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
sends := &sendsMade{}
moved, err := advanceOnce(withSendsMade(ctx, sends), open, p, edges, rollsOut)
if err != nil {
failed = true
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
// the state is left as it was and the step is tried again on the next tick. Said and
// the state is left as it was and the step is tried again by the next look. Said and
// kept when it is new: the same refusal on every tick is one fact, not one per tick.
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
@@ -757,14 +741,18 @@ func advanceHeld(ctx context.Context, open *stores) {
if planSnapshot(*p) != before {
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
failed = true
break
}
// Said once the send is kept (ADR 0406 §5, issue 974): what an outsider hears of a tier sent.
saySends(ctx, *p, sends)
}
if !moved {
break
}
}
}
return failed
}
// planSnapshot is a plan as a save would write it, to compare before and after a step: what the store
@@ -1066,6 +1054,16 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
p.Modules[m].SentAt = &now
p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat)
}
// One `sent` for each machine, with what it was sent (ADR 0406 §5, issue 974).
for _, n := range sent {
var carried []string
for _, m := range rest {
if slices.Contains(restTo[m], n) {
carried = append(carried, m)
}
}
noteSends(ctx, p.Tier, []string{n}, carried, now)
}
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
strings.Join(sent, ", "))
return true, nil
@@ -1176,7 +1174,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", "))
}
if len(silent) > 0 {
said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter,
said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", applySilentAfter,
strings.Join(silent, ", ")))
}
said = append(said, recordedSaid(*p)...)
@@ -1241,7 +1239,7 @@ func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time,
case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed):
failed = append(failed, node+" ("+outcome+")")
}
case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter:
case !at.IsZero() && now.Sub(at) > applySilentAfter:
seen[node] = true
silent = append(silent, node)
default:
@@ -1338,6 +1336,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
s.GatedBy = lead
}
}
noteSends(ctx, p.Tier, sent, modules, now)
p.State = inventory.PlanRolling
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "),
strings.Join(sent, ", "))
@@ -1652,23 +1651,6 @@ func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them"
}
// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
func planTicker(ctx context.Context, open *stores) {
advancePlans(ctx, open)
keepWalkPhases(ctx, open)
tick := time.NewTicker(30 * time.Second)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
advancePlans(ctx, open)
keepWalkPhases(ctx, open)
}
}
}
// planLine is one plan as `status` says it, given the least bound of a tier.
func planLine(p inventory.Plan, now time.Time) string {
return planLineWith(p, now, pauseView{}, tierAtLeast)
@@ -2034,7 +2016,7 @@ func plansCommand(ctx context.Context, args []string) error {
}); err != nil {
return err
}
fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked at the next pass\n", p.ID, p.Repository,
fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository,
short(p.Commit))
return nil
}
cmd/mesh-controller/rollout_events_test.go
+345
View File
@@ -0,0 +1,345 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// Acceptance of the step (novox/hq ADR 0406 §5, §6, acceptance 7; issue 974), live, with the store and a real bus and no
// ticker: a rollout of two tiers advances on the events alone — the plan kept, the machines' reports, a silent key's
// expiry — a machine that reports in time is never named silent and one that never reports is named silent when its
// key expires, and a controller replaced in the middle of it resumes with no tier asked twice and none skipped.
func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
// A quarter of an hour of silence is two seconds here.
wasSilent := applySilentAfter
applySilentAfter = 2 * time.Second
t.Cleanup(func() { applySilentAfter = wasSilent })
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
legacy, err := js.Conn().JetStream()
if err != nil {
t.Fatal(err)
}
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
t.Cleanup(func() { checkEvents = nil })
sentHeard := make(chan *nats.Msg, 32)
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
// Every save of a plan is kept, said and looked at by the serving controller's own hook.
saved := &movesSaid{revs: map[string][]int64{}}
wasSaved := inventory.PlanSaved
serving := servingPlanSaved(ctx, checkEvents, open)
inventory.PlanSaved = func(i *inventory.Inventory, p inventory.Plan) { saved.hear(i, p); serving(i, p) }
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
var mu sync.Mutex
var asks []string
var sends [][]string
wasAsk := askABuild
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
mu.Lock()
defer mu.Unlock()
id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond))
asks = append(asks, source.Repository+"#"+id)
return id, nil
}
t.Cleanup(func() { askABuild = wasAsk })
digests := 0
wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
mu.Lock()
defer mu.Unlock()
sends = append(sends, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
builds := map[string]string{}
for _, m := range []string{"a", "b"} {
builds[m] = current[m].Commit
}
for _, node := range names {
digests++
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
return nil, err
}
}
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
register := func(m, commit string, asked time.Time) {
t.Helper()
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
}
record := func(id, m, commit string, at time.Time) {
t.Helper()
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
old := time.Now().Add(-2 * time.Hour)
for _, m := range []string{"a", "b"} {
record("build-"+m+"-1", m, "c1", old)
register(m, "c1", old)
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, n, m); err != nil {
t.Fatal(err)
}
}
}
for _, n := range []string{"anchor", "laptop"} {
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
t.Fatal(err)
}
}
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
register("a", "c2", time.Now().Add(-time.Minute))
// A machine's report on what it was sent, heard as the controller hears it.
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
seq := int64(0)
reports := func(node string) {
t.Helper()
last, found, err := inv.SentTo(ctx, node)
if err != nil || !found {
t.Fatalf("%s was sent nothing: %v", node, err)
}
seq++
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
"report_sequence": seq})
var report link.Report
if err := json.Unmarshal(body, &report); err != nil {
t.Fatal(err)
}
if _, err := listener.Heard(ctx, report); err != nil {
t.Fatal(err)
}
}
heldKeys := func() map[silentWait]time.Time {
c := silentClockOf()
if c == nil {
return nil
}
held, err := c.Held(ctx)
if err != nil {
t.Fatal(err)
}
return held
}
plan := func() inventory.Plan {
t.Helper()
p, err := inv.PlanByID(ctx, "plan-974")
if err != nil {
t.Fatal(err)
}
return p
}
// The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything.
first, stopFirst := context.WithCancel(ctx)
defer stopFirst()
startRolloutEvents(first, open, js)
built := time.Now().UTC()
p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built,
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
Modules: map[string]*inventory.PlanModule{
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
"b": {},
}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
eventually(t, "tier 0 sent to both machines and tier 1 asked, on the save alone", func() bool {
mu.Lock()
defer mu.Unlock()
return len(sends) == 1 && len(asks) == 1 && plan().Tier == 1
})
eventually(t, "a key on the bus for each machine tier 0 was sent to", func() bool { return len(heldKeys()) == 2 })
// The controller is replaced: its looks and its watcher end. anchor reports meanwhile, to nobody.
stopFirst()
time.Sleep(300 * time.Millisecond)
reports("anchor")
second, stopSecond := context.WithCancel(ctx)
defer stopSecond()
startRolloutEvents(second, open, js)
// Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire.
eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool {
h := heldKeys()
_, anchor := h[silentWait{Plan: "plan-974", Tier: 0, Node: "anchor"}]
return !anchor
})
// tier 1's build comes in, as the build machine's outcome is taken in.
b := plan().Modules["b"]
if b == nil || b.Build == "" {
t.Fatalf("tier 1 was not asked: %+v", b)
}
askedAt, _ := link.BuildAskedAt(b.Build)
record(b.Build, "b", "c2", askedAt)
register("b", "c2", askedAt)
planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build)
eventually(t, "tier 1 sent to both machines", func() bool {
mu.Lock()
defer mu.Unlock()
return len(sends) == 2
})
reports("anchor")
// laptop never reports: its key expires and the new controller names it silent, which ends the rollout.
eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool {
p := plan()
return p.State == inventory.PlanDone
})
p = plan()
if !strings.Contains(p.Note, "laptop") || strings.Contains(p.Note, "anchor") || !strings.Contains(p.Note, "said nothing") {
t.Errorf("the plan ended with the note %q; want laptop named silent and not anchor", p.Note)
}
eventually(t, "no key left on the bus", func() bool { return len(heldKeys()) == 0 })
// No tier asked twice and none skipped: one ask for tier 1 (tier 0 was built when the plan was kept), each tier sent
// once to both machines, and said as sent once for each.
mu.Lock()
if len(asks) != 1 || !strings.HasPrefix(asks[0], "novox/b#") {
t.Errorf("asks = %v; want tier 1's module asked once", asks)
}
if len(sends) != 2 || strings.Join(sends[0], ",") != "anchor,laptop" || strings.Join(sends[1], ",") != "anchor,laptop" {
t.Errorf("sends = %v; want both machines sent each tier once", sends)
}
mu.Unlock()
said := map[string]int{}
for deadline := time.After(5 * time.Second); len(said) < 4; {
select {
case m := <-sentHeard:
var body planSentBody
if err := json.Unmarshal(m.Data, &body); err != nil {
t.Fatal(err)
}
said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++
case <-deadline:
t.Fatalf("sent was said for %v; want tiers 0 and 1 to both machines", said)
}
}
time.Sleep(300 * time.Millisecond)
for len(sentHeard) > 0 {
var body planSentBody
_ = json.Unmarshal((<-sentHeard).Data, &body)
said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++
}
for k, n := range said {
if n != 1 {
t.Errorf("sent for %s was said %d times", k, n)
}
}
// And every move of the plan was said as plan-moved.
saved.noUnsaid(t, open)
}
// Acceptance of the verb path (issue 974, review 1), live, with the store and a bus: a walk the delivery's owner lets
// go by the verb `deliver` after the operator's approval is asked its first tier with no other event. The verb runs as a
// child process, whose saves the serving controller does not hear; the handler asks the look as the verb ends.
func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
legacy, err := js.Conn().JetStream()
if err != nil {
t.Fatal(err)
}
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
t.Cleanup(func() { checkEvents = nil })
asked := asksRecorded(t)
register(t, open, catalogue.Manifest{Module: "app", Version: "1"})
if _, err := assign(ctx, open, "anchor", "app"); err != nil {
t.Fatal(err)
}
cut := time.Now().UTC()
waiting := inventory.Plan{ID: "plan-974-deliver", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8",
Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {}},
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat},
Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}}
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
t.Fatal(err)
}
// The serving controller, as serve starts it.
wasSaved := inventory.PlanSaved
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
started, stop := context.WithCancel(ctx)
defer stop()
startRolloutEvents(started, open, js)
time.Sleep(500 * time.Millisecond) // the first look: the walk waits, and is asked nothing
// The verb as its child process runs it: the delivery command, whose saves are not the serving controller's.
wasRun := runTheVerb
runTheVerb = func(ctx context.Context, argv []string) (verbAnswer, error) {
was := inventory.PlanSaved
inventory.PlanSaved = func(*inventory.Inventory, inventory.Plan) {}
defer func() { inventory.PlanSaved = was }()
return verbAnswer{}, deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"})
}
t.Cleanup(func() { runTheVerb = wasRun })
handlers, _, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
if _, err := handlers["deliver"](ctx, []byte(`{"plan":"`+waiting.ID+`","why":"its turn"}`)); err != nil {
t.Fatal(err)
}
eventually(t, "the walk's first tier asked, with no event but the verb's end", func() bool {
p, err := open.inventory.PlanByID(ctx, waiting.ID)
return err == nil && p.Modules["app"] != nil && p.Modules["app"].Build != "" && len(*asked) == 1
})
}
cmd/mesh-controller/rollout_silent_bus.go
+150
View File
@@ -0,0 +1,150 @@
package main
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// A rollout's wait for a machine is a key on the bus (novox/hq ADR 0406 §5, issue 974): each send to a machine puts
// a key `<plan>.<tier>.<node>` that lives as long as the machine may stay silent (ApplySilentAfter); the machine's
// report on that send deletes it, so only a missing report lets it expire; and the expiry, heard by a watcher,
// ends the wait for that machine, which the plan then names silent. No timer looks every so often whether a
// machine has reported.
// silentWait is one machine's wait on one send of a plan's tier.
type silentWait struct {
Plan string
Tier int
Node string
}
// silentKey is a wait's key. A plan id may hold a dot, so it is read from the right (parseSilentKey).
func silentKey(w silentWait) string { return w.Plan + "." + strconv.Itoa(w.Tier) + "." + w.Node }
// parseSilentKey reads a key back: the node is the last token, the tier the one before, the plan what is left.
func parseSilentKey(key string) (silentWait, bool) {
last := strings.LastIndexByte(key, '.')
if last < 1 || last == len(key)-1 {
return silentWait{}, false
}
rest, node := key[:last], key[last+1:]
mid := strings.LastIndexByte(rest, '.')
if mid < 1 || mid == len(rest)-1 {
return silentWait{}, false
}
tier, err := strconv.Atoi(rest[mid+1:])
if err != nil || tier < 0 {
return silentWait{}, false
}
return silentWait{Plan: rest[:mid], Tier: tier, Node: node}, true
}
// silentClock is the bus's side of the wait for a machine's report.
type silentClock interface {
// Arm puts a wait's key, to live ttl: made again if the machine is sent again. The key holds when the send was.
Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error
// Held is every wait that has a key now, with when its send was.
Held(ctx context.Context) (map[silentWait]time.Time, error)
// Reported takes a wait's key away: the machine reported on the send, and nothing is to wake the controller.
Reported(ctx context.Context, w silentWait) error
}
// busSilent is the clock on the controller's bucket.
type busSilent struct {
kv jetstream.KeyValue
}
// newBusSilent opens the bucket and starts the watcher, which names each wait whose key expired until the context
// ends. Only expiry wakes it: the marker of an expiry is a purge, and a key a report deleted, or one Arm made again,
// leaves a delete (the vendored client reads the server's MaxAge marker as a purge, kv.go:971).
func newBusSilent(ctx context.Context, api jetstream.JetStream, silent func(silentWait)) (*busSilent, error) {
kv, err := api.KeyValue(ctx, broker.RolloutSilentBucket)
if err != nil {
return nil, fmt.Errorf("the rollout's silent keys' bucket: %w", err)
}
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
return nil, fmt.Errorf("watching the rollout's silent keys: %w", err)
}
go func() {
defer w.Stop()
for {
select {
case <-ctx.Done():
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil || e.Operation() != jetstream.KeyValuePurge {
continue // a put, or a delete: a report's, or a send made again
}
if wait, ok := parseSilentKey(e.Key()); ok && silent != nil {
silent(wait)
}
}
}
}()
return &busSilent{kv: kv}, nil
}
func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error {
key := silentKey(w)
// Made again, not changed: a key's time to live is set when it is made.
if err := s.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
_, err := s.kv.Create(ctx, key, []byte(sent.UTC().Format(time.RFC3339Nano)), jetstream.KeyTTL(leastSecond(ttl)))
return err
}
func (s *busSilent) Reported(ctx context.Context, w silentWait) error {
if err := s.kv.Delete(ctx, silentKey(w)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
// leastSecond rounds a time to live up to a whole second, at least one: the server rounds a key's time down to
// whole seconds, so a fraction would close the key before the time it stands for.
func leastSecond(d time.Duration) time.Duration {
if d < time.Second {
return time.Second
}
return (d + time.Second - 1) / time.Second * time.Second
}
func (s *busSilent) Held(ctx context.Context) (map[silentWait]time.Time, error) {
keys, err := s.kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
if err != nil {
return nil, err
}
held := map[silentWait]time.Time{}
for _, key := range keys {
w, ok := parseSilentKey(key)
if !ok {
continue
}
e, err := s.kv.Get(ctx, key)
if err != nil {
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
return nil, err
}
at, _ := time.Parse(time.RFC3339Nano, string(e.Value()))
held[w] = at
}
return held, nil
}
cmd/mesh-controller/rollout_silent_bus_test.go
+120
View File
@@ -0,0 +1,120 @@
package main
import (
"context"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
// silentOnABus is the rollout's silent keys on a real bus, and the waits its watcher heard expire.
func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) {
t.Helper()
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
api, err := jetstream.New(conn)
if err != nil {
t.Fatal(err)
}
heard := make(chan silentWait, 16)
ctx, stop := context.WithCancel(context.Background())
t.Cleanup(stop)
s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w })
if err != nil {
t.Fatal(err)
}
return s, heard
}
func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (silentWait, bool) {
t.Helper()
select {
case w := <-heard:
return w, true
case <-time.After(within):
return silentWait{}, false
}
}
// A machine that never reports on its send is named silent when its key expires (issue 974, novox/hq ADR 0406 §5):
// the watcher names the plan, the tier and the machine, which a plan id or a machine name with a dot does not blur.
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) {
s, heard := silentOnABus(t)
want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {
t.Fatal(err)
}
if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want {
t.Fatalf("heard %+v (%v); want %+v", got, ok, want)
}
}
// A machine that reports in time is never named silent: the report deletes the key, and a delete is not an expiry.
func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) {
s, heard := silentOnABus(t)
ctx := t.Context()
w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"}
if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil {
t.Fatal(err)
}
if err := s.Reported(ctx, w); err != nil {
t.Fatal(err)
}
if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok {
t.Fatalf("%+v was named silent after it reported", got)
}
// Reporting twice, or on a send that put no key, is no error.
if err := s.Reported(ctx, w); err != nil {
t.Fatalf("a second report: %v", err)
}
}
// A send made again to a machine puts its key again with the full time, and only the last send's key can expire.
func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) {
s, heard := silentOnABus(t)
ctx := t.Context()
w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"}
if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil {
t.Fatal(err)
}
time.Sleep(1200 * time.Millisecond)
if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil {
t.Fatal(err)
}
if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok {
t.Fatalf("%+v expired at the first send's time, not the second's", got)
}
if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w {
t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w)
}
}
func TestASilentKeyIsParsedFromTheRightOfTheNode(t *testing.T) {
for _, w := range []silentWait{{"plan-1", 0, "ace"}, {"a.b", 12, "g14"}} {
got, ok := parseSilentKey(silentKey(w))
if !ok || got != w {
t.Errorf("%+v became %+v (%v)", w, got, ok)
}
}
for _, bad := range []string{"", "plan", "plan.x.ace", "plan..ace", ".1.ace"} {
if _, ok := parseSilentKey(bad); ok {
t.Errorf("%q was read as a wait", bad)
}
}
}
cmd/mesh-controller/seatverbs.go
+20 -3
View File
@@ -413,10 +413,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
if err := need("members"); err != nil {
return nil, err
}
argv := []string{"delivery", "check", "--members", str("members")}
if g := str("group"); g != "" {
return []string{"delivery", "check", "--group", g, "--members", str("members")}, nil
argv = []string{"delivery", "check", "--group", g, "--members", str("members")}
}
return []string{"delivery", "check", "--members", str("members")}, nil
// Marked ready (novox/hq ADR 0406 §3): the check is queued before every other check.
if r := strings.ToLower(str("ready")); r != "" && r != "false" && r != "no" && r != "0" {
argv = append(argv, "--ready")
}
return argv, nil
case "deliver":
if err := need("plan"); err != nil {
return nil, err
@@ -1398,12 +1403,24 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
link.Acknowledge(ctx)
}
return runVerb(ctx, argv)
answer, err := runTheVerb(ctx, argv)
// **A plan the verb moved is looked at as it ends** (novox/hq ADR 0406 §5, issue 974): the verb ran as a
// child process, whose saves this process does not hear, so `deliver` after an approval, a stop, a retry
// and a send from a verb would otherwise wait for an unrelated event. Even when it failed: it may have
// kept a plan before it did.
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
askToLook()
}
return answer, err
}
}
return handlers, behind, nil
}
// runTheVerb is how the serving controller runs a verb: as a command of its own. A variable so a test of what follows a
// verb needs no child process.
var runTheVerb = runVerb
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
func actsOnAPlan(args map[string]any) bool {
for _, act := range []string{"stop", "close", "retry"} {
cmd/mesh-controller/signals.go
+2
View File
@@ -671,6 +671,8 @@ func consumerWho(stream, consumer string) string {
switch {
case consumer == broker.ControllerName || consumer == broker.MergesConsumerName:
return "the controller"
case broker.IsCheckStream(stream):
return "the holder of " + broker.TheBuildSeat
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(consumer, "_worker"):
seat := strings.ToLower(strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(consumer, "SEAT_"), "_worker"), "_", "-"))
return "the holder of " + seat
cmd/mesh-controller/status_summary.go
+5
View File
@@ -210,6 +210,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
if news {
l.summary.nudge()
}
// A machine's account of an apply is looked at at once (novox/hq ADR 0406 §5, issue 974): it ends the wait for the
// machine, and the look is what deletes its key, knowing the report was on that send.
if err == nil && report.Declared != "" && report.Superseded == "" && report.Rekey == nil {
askToLook()
}
// What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions
// raised or cleared from it. Only from an account of the machine the store took.
if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" &&
cmd/mesh-controller/superseded.go
-157
View File
@@ -1,157 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// A new push to a pull request ends the check of its old head (novox/hq issue 726).
//
// A check judges one head; once the pull request has another, its verdict answers nobody and its run holds a build
// machine for minutes the new head waits behind. So when a check is asked for pull request N of a repository at a
// head, **every check of that pull request at another head ends**: a waiting one is dropped from the queue as
// `cancel` drops it, one in flight is ended on its machine as `kill` ends it. Each is said in the log.
//
// **Never anything else**: a try (`try`, asked for no pull request), a walk's build (no check), a delivery group's
// composed check (it answers several pull requests), another pull request's check, a dead ask (left for `clear
// --dead`), and a check of the same head — a recheck of that head is not another head.
// **Only for the pull request's current head.** Announcements of a pull request are not ordered, one is heard again
// after a restart, and a recheck asks for the head it recorded: a check asked for a head the pull request no longer
// has must never end the check of the head it has. So the forge is asked first which head the pull request has; only
// when it names this one does anything end, and when it cannot be asked, or names another, nothing ends and that is
// said.
// forgePullHead asks the forge, through its module's tools on the bus, which commit a pull request's head is. A
// variable so a test needs no forge.
var forgePullHead = func(ctx context.Context, js *broker.JetStream, owner, repo string, number int) (string, error) {
raw, _ := json.Marshal(map[string]any{"path": fmt.Sprintf("/repos/%s/%s/pulls/%d", owner, repo, number)})
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second)
if err != nil {
return "", err
}
if answer.Error != "" {
return "", errors.New(answer.Error)
}
var pull struct {
Result struct {
Head struct {
SHA string `json:"sha"`
} `json:"head"`
} `json:"result"`
}
if err := json.Unmarshal(answer.Result, &pull); err != nil {
return "", fmt.Errorf("the forge's answer about the pull request cannot be read: %w", err)
}
if pull.Result.Head.SHA == "" {
return "", errors.New("the forge named no head for the pull request")
}
return pull.Result.Head.SHA, nil
}
// supersededChecks are the checks in the queue that a check of owner/repo#number at head ends: of the same pull
// request, at another head, waiting or in flight, and not the ask keep.
func supersededChecks(q link.Queue, owner, repo string, number int, head, keep string) []link.QueuedAsk {
if number <= 0 || head == "" {
return nil
}
var out []link.QueuedAsk
for _, a := range q.Asks {
c := a.Request.Check
switch {
case c == nil, link.IsTry(a.Request), c.Group != "", c.Number != number, a.ID == keep,
!strings.EqualFold(c.Owner, owner), !strings.EqualFold(c.Repo, repo),
a.State != link.AskWaiting && a.State != link.AskInFlight:
continue
}
ref := a.Request.Ref
if ref == "" {
ref = a.Ref
}
if ref == "" || ref == head || link.SameHead(ref, head) {
continue
}
out = append(out, a)
}
return out
}
// endSupersededChecks ends the checks a check of p at its head supersedes, keep being the new check's own ask, and
// says each in the log. A waiting one is cancelled — nothing recorded: a check builds nothing, and the forge's status
// of the old head is no longer the pull request's — and one a machine builds is killed there.
func endSupersededChecks(ctx context.Context, js *broker.JetStream, seat string, p link.PullUpdated, keep string) []string {
if p.Number <= 0 || p.Commit == "" {
return nil
}
current, err := forgePullHead(ctx, js, p.Owner, p.Repo, p.Number)
if err != nil {
return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the forge could not say the pull request's "+
"current head, so checks of its other heads keep running: %v", p.Owner, p.Repo, p.Number, p.Commit, err)}
}
if current != p.Commit && !link.SameHead(current, p.Commit) {
return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the pull request's head is %.12s now, not "+
"this one", p.Owner, p.Repo, p.Number, p.Commit, current)}
}
q, err := link.ReadQueue(ctx, js, seat)
if err != nil {
return []string{fmt.Sprintf("%s/%s#%d (%.8s): could not read the %s queue for checks of its earlier heads: %v",
p.Owner, p.Repo, p.Number, p.Commit, seat, err)}
}
var said []string
for _, a := range supersededChecks(q, p.Owner, p.Repo, p.Number, p.Commit, keep) {
old := a.Request.Ref
if old == "" {
old = a.Ref
}
what := fmt.Sprintf("check of %s/%s#%d at %.12s", p.Owner, p.Repo, p.Number, old)
var err error
if a.State == link.AskInFlight && a.On != "" {
var answer link.Answer
answer, err = link.AskSeatTool(ctx, js.Conn(), seat, "kill", a.On, map[string]string{"id": a.ID}, killAnswer)
if err == nil && answer.Error != "" {
err = fmt.Errorf("%s refused: %s", a.On, answer.Error)
}
} else {
_, err = cancelAsk(ctx, js, nil, seat, a)
}
if err != nil {
said = append(said, fmt.Sprintf("not ended: %s (%s) — superseded by %.12s, but: %v", what, a.ID, p.Commit, err))
continue
}
said = append(said, fmt.Sprintf("ended: %s — superseded by %.12s (%s, %s)", what, p.Commit, a.ID, a.State))
}
return said
}
// endingSupersededChecks ends them beside the announcement, so a kill waiting on its machine never holds the
// handler: on its own connection to the bus, within twice a kill's wait. A one-shot process (oneShot) ends them
// within the call instead, as its exit would cut them off.
func endingSupersededChecks(ctx context.Context, seat string, p link.PullUpdated, keep string, oneShot bool) {
end := func() {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*killAnswer)
defer cancel()
js, err := dialTheBus()
if err != nil {
fmt.Fprintf(os.Stderr, "%s/%s#%d (%.8s): could not reach the bus to end checks of its earlier heads: %v\n",
p.Owner, p.Repo, p.Number, p.Commit, err)
return
}
defer js.Close()
for _, line := range endSupersededChecks(ctx, js, seat, p, keep) {
fmt.Println(line)
}
}
if oneShot {
end()
return
}
go end()
}
cmd/mesh-controller/superseded_test.go
-192
View File
@@ -1,192 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// A new push to a pull request ends the check of its old head (novox/hq issue 726): the waiting one is dropped from
// the queue, the one in flight is killed on its machine, and each is said. A try, a walk's build, another pull
// request's check, a group's check and a recheck of the new head are left alone. On the bus alone: no store.
func TestANewHeadEndsTheChecksOfTheOldOnes(t *testing.T) {
js, _ := aBuildQueue(t)
ctx := t.Context()
const (
oldHead = "1111111111111111111111111111111111111111"
olderHead = "2222222222222222222222222222222222222222"
newHead = "3333333333333333333333333333333333333333"
)
publish := func(r link.BuildRequest) {
t.Helper()
body, _ := json.Marshal(r)
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
t.Fatal(err)
}
}
check := func(id string, number int, head string) link.BuildRequest {
return link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: number}}
}
// In flight first: taken by a holder, started on ace.
publish(check("check-running", 7, olderHead))
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
t.Fatal(err)
}
started, _ := json.Marshal(link.BuildStart{ID: "check-running", On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
t.Fatal(err)
}
publish(check("check-old", 7, oldHead))
publish(check("check-other-pull", 8, oldHead))
publish(check("check-recheck", 7, newHead))
publish(check("check-new", 7, newHead))
try := check("try-old", 7, oldHead)
try.Check.Number, try.Check.Try = 0, true
publish(try)
group := check("group-old", 7, oldHead)
group.Check.Group = "g1"
publish(group)
publish(link.BuildRequest{ID: "walk-build", Repository: "https://forge.example/novox/a.git", Ref: oldHead})
var mu sync.Mutex
var killed []string
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", map[string]link.ToolHandler{
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
var args struct{ ID string }
_ = json.Unmarshal(raw, &args)
mu.Lock()
killed = append(killed, args.ID)
mu.Unlock()
return map[string]any{"said": "killed " + args.ID}, nil
}}, nil)
if err != nil {
t.Fatal(err)
}
defer stop()
forgeSays(t, newHead, nil)
said := endSupersededChecks(ctx, js, link.TheBuildMachine,
link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new")
mu.Lock()
if len(killed) != 1 || killed[0] != "check-running" {
t.Errorf("the holder was asked to kill %v, want only the in-flight check of the older head", killed)
}
mu.Unlock()
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
if err != nil {
t.Fatal(err)
}
if _, found := q.Find("check-old"); found {
t.Error("the waiting check of the old head is still in the queue")
}
for _, id := range []string{"check-other-pull", "check-recheck", "check-new", "try-old", "group-old", "walk-build"} {
if _, found := q.Find(id); !found {
t.Errorf("%s was ended; only checks of the same pull request at another head end", id)
}
}
text := strings.Join(said, "\n")
for _, want := range []string{
"ended: check of novox/a#7 at " + oldHead[:12] + " — superseded by " + newHead[:12],
"ended: check of novox/a#7 at " + olderHead[:12] + " — superseded by " + newHead[:12],
} {
if !strings.Contains(text, want) {
t.Errorf("the log does not say %q:\n%s", want, text)
}
}
if len(said) != 2 {
t.Errorf("said %d lines, want 2:\n%s", len(said), text)
}
}
// Which asks a check supersedes, read from the queue alone.
func TestSupersededChecksAreOnlyThePullRequestsOtherHeads(t *testing.T) {
ask := func(id, state, ref string, c *link.CheckRequest) link.QueuedAsk {
return link.QueuedAsk{ID: id, State: state, Ref: ref, Request: link.BuildRequest{ID: id, Ref: ref, Check: c}}
}
pr := func(n int) *link.CheckRequest { return &link.CheckRequest{Owner: "Novox", Repo: "A", Number: n} }
head := "abcdefabcdefabcdef0123456789abcdef012345"
q := link.Queue{Asks: []link.QueuedAsk{
ask("old", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
ask("short-same", link.AskWaiting, head[:12], pr(7)),
ask("dead", link.AskDead, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
ask("other", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(8)),
ask("build", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", nil),
ask("keep", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)),
}}
got := supersededChecks(q, "novox", "a", 7, head, "keep")
if len(got) != 1 || got[0].ID != "old" {
t.Fatalf("superseded %+v, want only old", got)
}
if got := supersededChecks(q, "novox", "a", 0, head, ""); got != nil {
t.Errorf("a check of no pull request superseded %+v", got)
}
}
// forgeSays makes the forge name head as every pull request's current head, or fail with err.
func forgeSays(t *testing.T, head string, err error) {
t.Helper()
was := forgePullHead
forgePullHead = func(context.Context, *broker.JetStream, string, string, int) (string, error) { return head, err }
t.Cleanup(func() { forgePullHead = was })
}
// An older head heard after a newer one — an announcement out of order, one heard again after a restart, a recheck
// of a recorded head — ends nothing: the forge says the pull request's head is the newer one. Nor does a check whose
// pull request's head the forge cannot say.
func TestAnOlderHeadHeardLateEndsNothing(t *testing.T) {
js, _ := aBuildQueue(t)
ctx := t.Context()
const (
oldHead = "1111111111111111111111111111111111111111"
newHead = "3333333333333333333333333333333333333333"
)
for _, id := range []string{"check-new", "check-old-late"} {
head := newHead
if id == "check-old-late" {
head = oldHead
}
body, _ := json.Marshal(link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: 7}})
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
t.Fatal(err)
}
}
late := link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: oldHead}
forgeSays(t, newHead, nil)
said := endSupersededChecks(ctx, js, link.TheBuildMachine, late, "check-old-late")
if len(said) != 1 || !strings.Contains(said[0], "nothing ended") {
t.Errorf("an older head heard late said %q", said)
}
forgeSays(t, "", errors.New("nothing serves gitea.gitea_api"))
said = endSupersededChecks(ctx, js, link.TheBuildMachine,
link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new")
if len(said) != 1 || !strings.Contains(said[0], "nothing ended") {
t.Errorf("a head the forge could not confirm said %q", said)
}
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
if err != nil {
t.Fatal(err)
}
for _, id := range []string{"check-new", "check-old-late"} {
if _, found := q.Find(id); !found {
t.Errorf("%s was ended", id)
}
}
}
cmd/mesh-controller/verb_looks_test.go
+54
View File
@@ -0,0 +1,54 @@
package main
import (
"context"
"sync/atomic"
"testing"
"time"
)
// A plan moved by a seat verb is looked at as the verb ends (issue 974, review 1): the verb runs as a child process
// whose saves the serving controller does not hear, so `deliver`, which lets a walk go after the operator's approval,
// would leave its first tier waiting for an unrelated event.
func TestAVerbThatMayMoveAPlanHasTheServingControllerLookWhenItEnds(t *testing.T) {
var looks, failures atomic.Int32
l := lookerCounting(&failures, &looks, nil)
lookerMu.Lock()
theLooker = l
lookerMu.Unlock()
t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
ctx, stop := context.WithCancel(context.Background())
defer stop()
go l.run(ctx)
was := runTheVerb
runTheVerb = func(context.Context, []string) (verbAnswer, error) { return verbAnswer{}, nil }
t.Cleanup(func() { runTheVerb = was })
handlers, _, err := seatToolHandlers()
if err != nil {
t.Fatal(err)
}
for _, c := range []struct {
verb string
args string
look bool
}{
{"deliver", `{"plan":"p"}`, true},
{"plans", `{"stop":"plan-1","why":"w"}`, true},
{"plans", `{}`, false},
{"nodes", `{}`, false},
} {
before := looks.Load()
h := handlers[c.verb]
if h == nil {
t.Fatalf("no verb %s", c.verb)
}
if _, err := h(ctx, []byte(c.args)); err != nil {
t.Fatalf("%s %s: %v", c.verb, c.args, err)
}
time.Sleep(100 * time.Millisecond)
if got := looks.Load() > before; got != c.look {
t.Errorf("%s %s: look asked = %v; want %v", c.verb, c.args, got, c.look)
}
}
}
cmd/mesh-controller/watchdogs.go
-3
View File
@@ -759,9 +759,6 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
// and the answer chosen is performed on its warrant.
startAsking(watching, open, server, bus.Conn, keeper)
go forgettingOldHeals(watching, open.inventory)
// And the build queue's order (novox/hq issue 558): a check that has waited its while goes ahead of the
// deliveries asked after it even when nothing new is asked.
go keepingTheQueueInOrder(watching, open, server.JetStream())
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
broker.ConditionsBucket, conditions.Seat)
internal/broker/checks.go
+118
View File
@@ -0,0 +1,118 @@
package broker
import "strings"
// The pull requests' checks have work queues of their own (novox/hq ADR 0406 §3, issue 974).
//
// A delivery build goes on the build seat's own queue, `mesh.seat.node-build-agent.accept.build`. A check of a pull
// request marked ready goes on a second work-queue stream and every other check on a third. A builder takes from the
// first, then the second, then the third; the others are taken in the order asked. **Each check stream has one subject
// per pull request and keeps only the newest request** (a limit of one message a subject, the older discarded), so a
// newer request replaces one still waiting and a pull request has at most one waiting check: the stream's rule, not a
// pass of the controller that moves asks about. Nothing pauses the worker or deletes and republishes a request.
const (
// ChecksReadyStream holds the checks of pull requests marked ready; ChecksOtherStream every other check.
ChecksReadyStream = "CHECKS_READY"
ChecksOtherStream = "CHECKS_OTHER"
// ChecksReadyWorker and ChecksOtherWorker are the durable consumers the build seat's holders share on them, as
// they share the seat's worker on its own queue (novox/hq ADR 0190).
ChecksReadyWorker = "CHECKS_READY_worker"
ChecksOtherWorker = "CHECKS_OTHER_worker"
// TheBuildSeat is the role whose checks these streams queue. The retired build role's checks stay on its own queue.
TheBuildSeat = "node-build-agent"
)
// PullsBucket is mesh-delivery's key per pull request: its head, the trunk commit its current check is asked against and
// its verdict (ADR 0406 §4). A running check watches its key.
var PullsBucket = BucketName("mesh-delivery", "pulls")
// CheckSubjectPrefix is where a seat's checks of one kind are asked: `mesh.seat.<seat>.check.ready.` or `.other.`. The
// `check` token keeps the subjects out of the seat's own queue, which takes `accept.>`.
func CheckSubjectPrefix(seat string, ready bool) string {
kind := "other"
if ready {
kind = "ready"
}
return "mesh.seat." + seat + ".check." + kind + "."
}
// CheckStreams are the two check queues, in the order a builder takes from them.
func CheckStreams() []Stream {
return []Stream{
{
Name: ChecksReadyStream,
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">"},
Retention: RetentionWorkQueue,
MaxAge: 7 * 24 * 60 * 60,
MaxMsgsPerSubject: 1,
Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " +
"(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " +
"request still waiting",
},
{
Name: ChecksOtherStream,
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, false) + ">"},
Retention: RetentionWorkQueue,
MaxAge: 7 * 24 * 60 * 60,
MaxMsgsPerSubject: 1,
Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " +
"request, the newest request kept; one left behind for a head already judged is dropped when taken",
},
}
}
// CheckWorkers are the consumers the build seat's holders pull the check queues through: as the seat's own worker is,
// one for every holder, one ask at a time, acknowledged when the check is done.
func CheckWorkers() []Consumer {
worker := func(name, stream string, ready bool) Consumer {
return Consumer{
Name: name,
Stream: stream,
Filters: []string{CheckSubjectPrefix(TheBuildSeat, ready) + ">"},
AckWaitSeconds: 60,
MaxDeliver: 5,
Why: "every holder of " + TheBuildSeat + " pulls one check at a time from this worker and acknowledges after " +
"the check is done, so a crash mid-check redelivers rather than loses (novox/hq ADR 0406 §3)",
}
}
return []Consumer{
worker(ChecksReadyWorker, ChecksReadyStream, true),
worker(ChecksOtherWorker, ChecksOtherStream, false),
}
}
// holderCheckGrants are what a holder of the build seat is granted to take checks: ask about the two workers, pull from
// them and acknowledge, and nothing of the subjects themselves.
func holderCheckGrants() []string {
var pub []string
for _, w := range CheckWorkers() {
pub = append(pub,
"$JS.API.CONSUMER.INFO."+w.Stream+"."+w.Name,
"$JS.API.CONSUMER.MSG.NEXT."+w.Stream+"."+w.Name,
"$JS.ACK."+w.Stream+"."+w.Name+".>")
}
// And the bucket of the checks they are running: each writes the key of its own check, reads another's, and the bucket
// is a controller's own bucket the build seat's holders write by design (ADR 0406 §3).
pub = append(pub,
"$KV."+CheckRunningBucket+".>",
"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
return pub
}
// IsCheckStream says a stream is one of the check queues.
func IsCheckStream(stream string) bool {
return stream == ChecksReadyStream || stream == ChecksOtherStream
}
// isCheckAsk says a message of a check queue is an ask the controller itself makes.
func isCheckAsk(stream, subject string) bool {
switch stream {
case ChecksReadyStream:
return strings.HasPrefix(subject, CheckSubjectPrefix(TheBuildSeat, true))
case ChecksOtherStream:
return strings.HasPrefix(subject, CheckSubjectPrefix(TheBuildSeat, false))
}
return false
}
internal/broker/checks_test.go
+42
View File
@@ -0,0 +1,42 @@
package broker
import "testing"
// The build agent's credential is the module principal `<node>.build-agent` (Username, KindModule): its manifest's
// `reads: ["mesh-delivery.pulls"]` reaches that principal as a read and a watch of mesh-delivery's bucket of pull requests'
// keys — the direct get of a key, the consumer a watch makes, the stream's info — and as no write and no direct
// subscription of the bucket's subjects (novox/hq ADR 0406 §4, issue 974).
func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat},
Reads: []string{"mesh-delivery.pulls"}, PasswordHash: "x"}
if got := p.Username(); got != "one.build-agent" {
t.Fatalf("the build agent's credential is %q", got)
}
perms, err := PermissionsFor(p)
if err != nil {
t.Fatal(err)
}
for _, s := range []string{
"$JS.API.STREAM.INFO.KV_mesh-delivery_pulls",
"$JS.API.DIRECT.GET.KV_mesh-delivery_pulls.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery_pulls.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery_pulls.>",
// And the two check workers it takes from.
"$JS.API.CONSUMER.MSG.NEXT.CHECKS_READY.CHECKS_READY_worker",
"$JS.API.CONSUMER.MSG.NEXT.CHECKS_OTHER.CHECKS_OTHER_worker",
"$JS.ACK.CHECKS_OTHER.CHECKS_OTHER_worker.>",
// And the bucket of the checks running, which it writes and reads (ADR 0406 §3).
"$KV." + CheckRunningBucket + ".>",
"$JS.API.DIRECT.GET.KV_" + CheckRunningBucket + ".>",
} {
has(t, perms.Publish, s)
}
hasNot(t, perms.Publish, "$KV."+PullsBucket+".>")
hasNot(t, perms.Publish, "mesh.seat.node-build-agent.check.>")
for _, s := range perms.Subscribe {
if s == "$KV."+PullsBucket+".>" {
t.Fatalf("the build agent subscribes the bucket's subjects directly: %v", perms.Subscribe)
}
}
}
internal/broker/controller_buckets.go
+58 -2
View File
@@ -43,6 +43,19 @@ var (
// `<batch>.most`, made once when it opens and living the window's most. The first to expire closes the
// batch, and the controller's watcher hears the delete marker. Written by the lease holder alone.
MergeWindowBucket = BucketName(ControllerSeat, "merge-window")
// RolloutSilentBucket holds the wait of every machine a rollout has sent a tier to (novox/hq ADR 0406 §5, issue
// 974): per send a key `<plan>.<tier>.<node>` that lives ApplySilentAfter, made when the controller sends that
// machine and deleted by the machine's report on that send. Only a missing report lets it expire, and the
// controller's watcher hearing the expiry ends the wait and names the machine silent. Written by the lease
// holder alone.
RolloutSilentBucket = BucketName(ControllerSeat, "rollout-silent")
// CheckRunningBucket holds the pull request checks the builders are running now (novox/hq ADR 0406 §3, issue 974): per
// pull request a key `<owner>.<repo>.<number>` (or `group.<name>`) holding the head and trunk commit being checked, the
// builder and its build. A builder puts it when it starts a check and again on each heartbeat, and deletes it when the
// check ends; a key not renewed within the bucket's age is a builder that died, and is gone. A builder that takes a
// waiting request for a head and trunk commit a key names drops it. **Written by the build seat's holders, several by
// design** — each the key of the check it runs.
CheckRunningBucket = BucketName(ControllerSeat, "checks-running")
)
// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
@@ -88,6 +101,18 @@ const (
// that is up to hear it, short enough to leave nothing behind.
const MergeWindowMarkerTTL = time.Minute
// RolloutSilentMarkerTTL is how long the marker an expired silent key leaves is kept, for the same reason.
const RolloutSilentMarkerTTL = time.Minute
// RolloutSilentKeptFor is the age of the silent keys' bucket: every report deletes a key and the delete leaves a marker, which
// with one value per key would stay for ever and fill the bucket. A key lives a quarter of an hour at most, so an hour
// takes nothing live.
const RolloutSilentKeptFor = time.Hour
// CheckRunningKeptFor is the age of a running check's key: a few heartbeats (ten seconds each) of a builder, so a builder that
// died is forgotten within two minutes and the check it ran is taken again then.
const CheckRunningKeptFor = 2 * time.Minute
// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
const AskedKeptFor = 30 * 24 * time.Hour
@@ -116,13 +141,13 @@ func IsControllerBucket(bucket string) bool {
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
bucket == MergeWindowBucket
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket
}
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
func ControllerBuckets() []string {
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket}
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket}
}
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
@@ -234,6 +259,37 @@ func (j *JetStream) EnsureControllerBuckets() error {
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", MergeWindowBucket, err)
}
// **The rollout's silent keys: keys that expire** (ADR 0406 §5, issue 974). A key lives as long as the machine
// may stay silent; the machine's report deletes it, and a marker of expiry is what the watcher hears.
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: RolloutSilentBucket,
Description: "the wait of every machine a rollout sent a tier to (novox/hq ADR 0406): `<plan>.<tier>.<node>` " +
"lives the time a machine may stay silent and is deleted by its report; one that expires names the " +
"machine silent; written by the controller alone",
History: 1,
TTL: RolloutSilentKeptFor,
MaxValueSize: 4 << 10,
MaxBytes: 8 << 20,
Storage: jetstream.FileStorage,
LimitMarkerTTL: RolloutSilentMarkerTTL,
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", RolloutSilentBucket, err)
}
// **The checks running: a key that ages** (ADR 0406 §3, issue 974). One message a key, each renewal a new one, so the
// age of the bucket is how long a key outlives the builder that stopped renewing it.
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: CheckRunningBucket,
Description: "the pull request checks the builders are running now (novox/hq ADR 0406): `<pull request>` holds the " +
"head and trunk commit, the builder and its build; put by the builder when it starts and at each heartbeat, " +
"deleted when the check ends, gone within two minutes of a builder that died; written by the build seat's holders",
History: 1,
TTL: CheckRunningKeptFor,
MaxValueSize: 4 << 10,
MaxBytes: 8 << 20,
Storage: jetstream.FileStorage,
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
}
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
for _, m := range []struct{ bucket, why string }{
internal/broker/controller_buckets_test.go
+103
View File
@@ -322,6 +322,109 @@ func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) {
}
}
// **A rollout's wait for a machine is a bucket whose keys expire** (novox/hq ADR 0406 §5, issue 974): a key made with a
// time to live goes when it is up, and a watcher hears it go as a purge; a key a report deleted is a delete. The
// controller may write it, and it is no undeclared state.
func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) {
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") {
t.Error("the controller may not write the rollout's silent keys' bucket")
}
if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) {
t.Error("the rollout's silent keys' bucket is not asserted as the controller's own")
}
js, err := Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer js.Close()
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
kv, err := api.KeyValue(ctx, RolloutSilentBucket)
if err != nil {
t.Fatal(err)
}
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
t.Fatal(err)
}
defer w.Stop()
if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil {
t.Fatal(err)
}
if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil {
t.Fatal(err)
}
if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
t.Fatal(err)
}
for {
select {
case e := <-w.Updates():
if e == nil || e.Operation() == jetstream.KeyValuePut {
continue
}
if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete {
t.Fatalf("a report's delete was heard as %v", e.Operation())
}
if e.Key() == "plan-1.0.g14" {
if e.Operation() != jetstream.KeyValuePurge {
t.Fatalf("an expiry was heard as %v, not a purge", e.Operation())
}
return
}
case <-ctx.Done():
t.Fatal("no watcher heard the key expire")
}
}
}
// **The silent keys' bucket ages**, or its delete markers would fill it (issue 974, review 1): every report deletes a key,
// the delete leaves a marker, and with one value per key and no age the markers stay for ever. A bucket made before the
// age was set is brought to it by the next start, which the server accepts as an update of an existing bucket.
func TestTheSilentKeysBucketAgesAndAnOlderOneIsBroughtToIt(t *testing.T) {
js, err := Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer js.Close()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
// As the first build made it: no age.
if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: RolloutSilentBucket, History: 1,
MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, LimitMarkerTTL: RolloutSilentMarkerTTL}); err != nil {
t.Fatal(err)
}
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatalf("a bucket made without the age is not brought to it: %v", err)
}
kv, err := api.KeyValue(ctx, RolloutSilentBucket)
if err != nil {
t.Fatal(err)
}
status, err := kv.Status(ctx)
if err != nil {
t.Fatal(err)
}
if status.TTL() != RolloutSilentKeptFor {
t.Errorf("the bucket ages at %v; want %v", status.TTL(), RolloutSilentKeptFor)
}
}
// **The merges have a consumer of their own, which a reset of the events consumer leaves alone** (issue 973, novox/hq
// ADR 0407 decision 3): healer H4 re-makes the events consumer from now, dropping what it holds pending; a merge
// waiting in the merges consumer is not that consumer's, and nothing would catch it up.
internal/broker/nats.go
+33
View File
@@ -217,6 +217,9 @@ var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"}
// original — and no other can (novox/hq issue 334, ADR 0264's consequences: no grant over the seats'
// queues).
func TheControllersAsk(stream, subject string) bool {
if isCheckAsk(stream, subject) {
return true
}
for _, seat := range seatsTheControllerAsks {
if stream == seatStreamName(seat) && strings.HasPrefix(subject, "mesh.seat."+seat+".accept.") {
return true
@@ -378,6 +381,27 @@ var DegradableGrants = []DegradableGrant{
Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " +
"window setting says (ADR 0407 decision 1, issue 973)",
},
{
Subject: "mesh.seat." + TheBuildSeat + ".check.>",
Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
"first push, and a check asked before that is asked again by the next announcement of its head (ADR 0406 §3, " +
"issue 974)",
},
{
Subject: "$KV." + CheckRunningBucket + ".>",
Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " +
"holders do (ADR 0406 §3, issue 974)",
},
{
Subject: "mesh.seat." + ControllerSeat + ".event.sent",
Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
"be said is printed and goes on (ADR 0406 §5, issue 974)",
},
{
Subject: "$KV." + RolloutSilentBucket + ".>",
Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " +
"for that wait (ADR 0406 §5, issue 974)",
},
}
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
@@ -565,6 +589,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
// the role, and whichever machine holding it is idle takes it.
for _, seat := range seatsTheControllerAsks {
pub = append(pub, "mesh.seat."+seat+".accept.>")
// **And its checks, on the two queues of their own** (novox/hq ADR 0406 §3, issue 974): one subject a pull
// request on each, under `check`; the retired role's checks stay on its accepts.
if seat == TheBuildSeat {
pub = append(pub, "mesh.seat."+seat+".check.>")
}
// **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is
// building, kill it, pause it, resume it. The queue is the controller's to show and to
// change, and what one machine is doing with an ask it took only that machine can say.
@@ -873,6 +902,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
"$JS.ACK."+stream+"."+worker+".>")
// And the checks' two workers, for the build role they are queued for (novox/hq ADR 0406 §3).
if s.Name == TheBuildSeat {
pub = append(pub, holderCheckGrants()...)
}
// **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the
// seat's cancelled set, read directly by its id, so a holder that fetched an ask in the
// moment the controller cancelled it ends it instead of building it. Read, never written:
internal/broker/states_agreement_test.go
+2
View File
@@ -34,6 +34,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
states = append(states, link.KeyChecked)
// And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
states = append(states, link.KeyPlanMoved)
// And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
states = append(states, link.KeySent)
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged, link.KeyPredefinedChanged)
for _, event := range states {
internal/broker/streams.go
+6 -4
View File
@@ -147,7 +147,7 @@ func OriginalOfAgain(subject string) (string, bool) {
// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps
// tools on core NATS, where a lost call is a timeout the caller already handles).
func MeshStreams() []Stream {
return []Stream{
return append([]Stream{
{
Name: "CONTROL",
// A build's outcome is no longer here: it is the build-machine seat's own event, so one
@@ -221,7 +221,7 @@ func MeshStreams() []Stream {
"failed with the reason (novox/hq ADR 0349 §4): what a walk's end and a gate wait for (ADR 0362, " +
"ADR 0349 §15), read by subject; a week, as the events are",
},
}
}, CheckStreams()...)
}
// ProvAppliedStream keeps what each machine's provisioner client last said of each module's files, newest per
@@ -368,6 +368,8 @@ var ControllerStates = []string{"applied", "refused", "built-before",
"checked",
// And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from.
"plan-moved",
// And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
"sent",
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
// each registered module's declared config (design 57 F3).
"assignment-changed", "declared-config-changed", "predefined-changed"}
@@ -463,7 +465,7 @@ const EventsStream = "EVENTS"
// held through a store restart — the exact message the stream exists to protect — some minutes
// before the controller had finished deciding about it.
func MeshConsumers() []Consumer {
return []Consumer{
return append([]Consumer{
{
Name: ControllerName,
Stream: "CONTROL",
@@ -523,7 +525,7 @@ func MeshConsumers() []Consumer {
Why: "the controller copies each message a consumer gave up on into DEAD_LETTERS; no max-deliver, " +
"because a notice it gave up on would lose the message it is about",
},
}
}, CheckWorkers()...)
}
// NoticesConsumer is the controller's consumer on DEAD_LETTER_NOTICES (novox/hq issue 330).
internal/broker/streams_test.go
+3
View File
@@ -134,6 +134,9 @@ func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) {
// What each machine's provisioner client last said of each module's files (design 57 P6): the newest
// word per module, what a walk's end reads.
"PROV_APPLIED": RetentionLastPerSubject,
// The checks of pull requests, in two work queues of their own (novox/hq ADR 0406 §3, issue 974).
"CHECKS_READY": RetentionWorkQueue,
"CHECKS_OTHER": RetentionWorkQueue,
}
got := map[string]Retention{}
for _, s := range MeshStreams() {
internal/broker/testdata/composed.conf
Vendored
+1 -1
View File
@@ -24,7 +24,7 @@ accounts {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
allow_responses: { max: 1, ttl: "1m" }
} }
internal/broker/writers.go
+23 -4
View File
@@ -41,6 +41,14 @@ func ownMachine(p Principal, pattern string) bool {
return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
}
// holdsTheBuildSeat is a principal holding the build seat, or the controller, which asserts the bucket the holders write.
func holdsTheBuildSeat(p Principal, _ string) bool {
if p.Kind == KindController {
return true
}
return holdsSeatOf(p, "mesh.seat."+TheBuildSeat+".x")
}
// holdsSeatOf is a principal holding the seat a `mesh.seat.<seat>.…` pattern names: its module's own
// principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules).
func holdsSeatOf(p Principal, pattern string) bool {
@@ -117,11 +125,13 @@ var WritersTable = []WriterRow{
Subjects: []string{"mesh.control.*.report", "mesh.control.*.health", "mesh.control.*.cli"}, Writes: ownMachine},
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
// And the clock of an open batch's merge window, a plan before it is one (novox/hq ADR 0407 decision 1): keys that
// expire in a bucket of their own, written by the lease holder.
// And the clock of an open batch's merge window, a plan before it is one (novox/hq ADR 0407 decision 1), and the
// wait of a rollout for each machine it sent (ADR 0406 §5, issue 974): keys that expire in buckets of their own,
// written by the lease holder.
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket, Others: "read through plans",
Subjects: kvOf(MergeWindowBucket), Writes: isController},
KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket +
", a rollout's wait for a machine's report in key-value " + RolloutSilentBucket, Others: "read through plans",
Subjects: append(kvOf(MergeWindowBucket), kvOf(RolloutSilentBucket)...), Writes: isController},
{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " +
"controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads",
@@ -146,6 +156,15 @@ var WritersTable = []WriterRow{
Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
Writes: holdsSeatOf,
Shared: "every machine holding the build seat answers the asks it took; each outcome names its ask"},
{State: "a pull request's check asked", Writer: "controller",
KeptIn: "the work queues " + ChecksReadyStream + " and " + ChecksOtherStream + ", one subject a pull request, the newest kept",
Others: "the build seat's holders take it",
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">", CheckSubjectPrefix(TheBuildSeat, false) + ">"},
Writes: isController},
{State: "a pull request's check running", Writer: "the build seat's holders — several by design, each the key of the check it runs",
KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request",
Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat,
Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"},
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
internal/broker/writers_test.go
+2
View File
@@ -27,6 +27,8 @@ var designRows = []string{
"the healers' acts and their brake",
"stream definitions and bus permissions",
"builds and their outcomes",
"a pull request's check asked",
"a pull request's check running",
"a merge announced",
"a pull request's head announced",
"a pull request's merge check",
internal/catalogue/seats.go
+2
View File
@@ -108,6 +108,8 @@ var defaultSeats = append([]Seat{
"checked",
// A walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
"plan-moved",
// A tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
"sent",
// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
"assignment-changed", "declared-config-changed", "predefined-changed"},
Serves: ControllerVerbs},
internal/catalogue/streams.go
+1 -1
View File
@@ -36,7 +36,7 @@ var ReservedStreamPrefixes = []string{"SEAT_", "LOG_", "KV_", "OBJ_", "DEAD_"}
// MeshStreamNames are the mesh's own streams by name. Written here as well as in the broker, which this
// package cannot import; a test keeps them agreeing.
var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED"}
var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED", "CHECKS_READY", "CHECKS_OTHER"}
// StreamDeclaration is one stream a module owns.
type StreamDeclaration struct {
internal/catalogue/verbs.go
+3 -1
View File
@@ -160,7 +160,9 @@ var ControllerVerbs = []Verb{
"the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " +
"again as its pull request is.",
Input: schema(map[string]string{"group": "the group's id (none for one head checked again)",
"members": "the members, as JSON, as delivery-order takes them"}, []string{"members"})},
"members": "the members, as JSON, as delivery-order takes them",
"ready": "\"true\" when the pull request is marked ready: its check is queued before every other check (novox/hq ADR 0406 §3)"},
[]string{"members"})},
{Name: "deliver", Description: "The delivery's word that a walk waiting for it may start (novox/hq ADR 0239): its " +
"first tier is asked at the controller's next pass. Refused for a walk that waits for nobody.",
Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"},
internal/link/advisories.go
+3 -1
View File
@@ -143,7 +143,7 @@ func MeshNamed(stream, name string) bool {
return false // the mesh defines no durable consumer on a bucket's stream, or the memberships'
case stream == "NODES":
return true
case strings.HasPrefix(stream, "SEAT_"):
case strings.HasPrefix(stream, "SEAT_") || broker.IsCheckStream(stream):
return strings.HasSuffix(name, "_worker")
case name == broker.ControllerName || name == broker.MergesConsumerName:
return true
@@ -160,6 +160,8 @@ func ConsumerInWords(stream, name string) string {
return "the controller's consumer on " + stream
case stream == "NODES":
return "how " + name + " hears what it should be (its declaration consumer)"
case broker.IsCheckStream(stream):
return "the worker every holder of " + broker.TheBuildSeat + " takes its checks from (" + stream + ")"
case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"):
seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-"))
return "the worker every holder of " + seat + " takes its asks from"
internal/link/build.go
+6
View File
@@ -144,6 +144,12 @@ type CheckRequest struct {
Repo string `json:"repo"`
Number int `json:"number,omitempty"`
Base string `json:"base,omitempty"`
// Trunk is the base branch's head the check is asked against, as the forge announced it with the head (novox/hq ADR
// 0406 §4): with the head, what the pull request's key says the current check is for, and what a waiting request
// is dropped by and a running check stopped by when the key moves.
Trunk string `json:"trunk,omitempty"`
// Ready says the pull request is marked ready (ADR 0406 §3): its check is queued before every other check.
Ready bool `json:"ready,omitempty"`
// Paths are the files the pull request changes, for the width of its rebuild.
Paths []string `json:"paths,omitempty"`
// Beside are the repositories the check reads next to this one, each cloned at the ref given — the
internal/link/builds_nats.go
+208 -74
View File
@@ -76,11 +76,9 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
}
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
if _, err := b.js.Context().Publish(WorkSubjectOf(b.role(), request), body, nats.Context(publish)); err != nil {
return fmt.Errorf("cannot submit a build: %w", err)
}
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
signalAsked()
return nil
}
@@ -112,11 +110,9 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
// be assumed, because nothing else will ever say so.
publish, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
if _, err := b.js.Context().Publish(WorkSubjectOf(b.role(), request), body, nats.Context(publish)); err != nil {
return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err)
}
// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
signalAsked()
waiting, cancelWait := context.WithTimeout(ctx, wait)
defer cancelWait()
@@ -162,10 +158,29 @@ type natsMachine struct {
js *broker.JetStream
on string
seat string
sub *nats.Subscription
// subs are the queues this machine takes from, in the order it takes from them: the seat's own, where a delivery
// build waits, then the checks of pull requests marked ready, then every other check (novox/hq ADR 0406 §3).
subs []*takeQueue
opts MachineOptions
}
// takeQueue is one queue a machine pulls from.
type takeQueue struct {
sub *nats.Subscription
// kind is what its asks are: a build, a ready check, or another check.
kind string
// worker is the consumer bound; bound is false until a bind succeeded, for a queue the mesh may not have made yet.
worker broker.Consumer
bound bool
}
// The kinds of queue a machine takes from, in the order it takes from them.
const (
queueBuilds = "builds"
queueReady = "ready"
queueOther = "other"
)
// MachineOptions is what a holder tells the taking loop about itself (novox/hq ADR 0219).
type MachineOptions struct {
// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
@@ -207,47 +222,56 @@ const (
)
func (m *natsMachine) Close() {
if m.sub != nil {
_ = m.sub.Unsubscribe()
for _, q := range m.subs {
if q.sub != nil {
_ = q.sub.Unsubscribe()
}
}
}
// Take binds to the role's worker and pulls one request at a time, handing each over.
// Take binds to the role's workers and pulls one request at a time, handing each over.
//
// **Bound, never created.** The work queue and the worker on it are the controller's to define
// **Bound, never created.** The work queues and the workers on them are the controller's to define
// (design 25 §3), and a build machine reaches no part of the JetStream API — so a missing one is said
// as the mesh's to answer rather than quietly created with whatever this client defaults to.
//
// **Pulled, one at a time, by whichever holder is free** (novox/hq ADR 0190). Every machine holding
// the role binds this same worker; a machine asks for the next request only when it has finished
// the role binds these same workers; a machine asks for the next request only when it has finished
// the last, so a slow machine never holds an ask an idle one could take, and a machine that took
// five at once would run five container builds against one runtime and finish all of them slower
// than the first.
//
// **In this order** (novox/hq ADR 0406 §3): a delivery build, then a check of a pull request marked ready, then any
// other check, each queue tried without waiting; when all three are empty the first is waited on for a second at most
// and the three are tried again, so a delivery build is never much more than a second behind and nothing spins.
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
worker, found := broker.HolderConsumerFor(m.on, "build-agent",
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
if !found {
return fmt.Errorf("%s accepts no work, so there is nothing for this machine to take", m.seat)
}
// **The consumer's own filter, not the one subject this machine cares about.** The client checks
// what is asked for against the consumer's filter and refuses anything that is not the same —
// "subject does not match consumer" — so subscribing `…accept.build` against a consumer filtered
// on `…accept.>` is rejected even though it is narrower. Learned twice now, on two different
// consumers, which is why it is written down here.
filter := worker.Filters[0]
sub, err := m.js.Context().PullSubscribe(filter, worker.Name,
nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
m.subs = []*takeQueue{{kind: queueBuilds, worker: worker}}
if m.seat == broker.TheBuildSeat {
for i, w := range broker.CheckWorkers() {
kind := queueReady
if i == 1 {
kind = queueOther
}
m.subs = append(m.subs, &takeQueue{kind: kind, worker: w})
}
}
// The seat's own queue is bound at once and its absence is this machine's error, as it always was. The check queues
// are bound when they are there: a controller that has not made them yet is no reason to take no build.
if err := m.bind(m.subs[0]); err != nil {
return fmt.Errorf(
"this machine cannot take work from %s: %w. The mesh creates that queue and this "+
"machine's worker on it, and a build machine may not create one itself — so this is "+
"the mesh's to answer, not this machine's", m.seat, err)
}
m.sub = sub
if m.opts.Taking != nil {
m.opts.Taking()
}
var bindAgain time.Time
for {
if ctx.Err() != nil {
@@ -264,29 +288,31 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
}
continue
}
// One, and wait a while for it; an empty queue is a timeout, which is the normal state of a
// machine with nothing to build, and is asked again.
// Asked for a few seconds at a time when the holder can be paused, so a pause reaches a pull
// already waiting within that, rather than when the client's own wait runs out.
asking, endAsking := ctx, func() {}
if m.opts.Paused != nil {
asking, endAsking = context.WithTimeout(ctx, pausedFetch)
if time.Now().After(bindAgain) {
for _, q := range m.subs[1:] {
if !q.bound {
if err := m.bind(q); err != nil {
fmt.Fprintf(os.Stderr, "the %s check queue is not there to take from yet: %v\n", q.kind, err)
}
}
}
bindAgain = time.Now().Add(30 * time.Second)
}
fetched, err := sub.Fetch(1, nats.Context(asking))
endAsking()
msg, q, err := m.next(ctx)
switch {
case ctx.Err() != nil:
// Ours ended: the machine is being stopped.
return nil
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
case msg != nil:
case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout), err == nil:
// **An empty queue, not the end.** A fetch on a context without a deadline waits the
// client's own while and then says the deadline passed — the client's, not ours. Read
// as "stop", every idle build machine exited clean every half minute and was started
// again by its supervisor, which looked like a crash loop with nothing in the log to
// say why (2026-10-03, the first build agents). Asked again.
continue
case err != nil:
if sub.IsValid() {
default:
if m.subs[0].sub.IsValid() {
// A transient fault in asking — a reconnect, a slow server — is asked past rather
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
time.Sleep(time.Second)
@@ -294,49 +320,151 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
}
return fmt.Errorf("the bus stopped delivering build work: %w", err)
}
for _, msg := range fetched {
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
// the one that arrived in that moment. Handed back at once for another holder — counted as
// a delivery, which a pause landing exactly then costs and nothing else does.
if m.opts.Paused != nil && m.opts.Paused() {
_ = msg.Nak()
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
// the one that arrived in that moment. Handed back at once for another holder — counted as
// a delivery, which a pause landing exactly then costs and nothing else does.
if m.opts.Paused != nil && m.opts.Paused() {
_ = msg.Nak()
continue
}
var request BuildRequest
if err := json.Unmarshal(msg.Data, &request); err != nil {
// Unreadable: terminated rather than retried, because the next attempt reads the same
// bytes. Nobody waiting hears an answer, which is right — there was no request.
_ = msg.Term()
continue
}
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
// id into the seat's cancelled set before deleting the ask, so one taken in between is
// ended here, terminated rather than redelivered, and its outcome said as failed — never
// built. A set that cannot be read is said and the ask built: a cancel is a person's
// exception, and a holder that refused every build while its grant was missing would
// stop the mesh building for it.
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
} else if cancelled {
endCancelled(ctx, build)
continue
}
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
if q.kind == queueOther {
if why := m.judgedAlready(ctx, request); why != "" {
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
_ = msg.Ack()
continue
}
var request BuildRequest
if err := json.Unmarshal(msg.Data, &request); err != nil {
// Unreadable: terminated rather than retried, because the next attempt reads the same
// bytes. Nobody waiting hears an answer, which is right — there was no request.
_ = msg.Term()
continue
}
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
// id into the seat's cancelled set before deleting the ask, so one taken in between is
// ended here, terminated rather than redelivered, and its outcome said as failed — never
// built. A set that cannot be read is said and the ask built: a cancel is a person's
// exception, and a holder that refused every build while its grant was missing would
// stop the mesh building for it.
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
} else if cancelled {
endCancelled(ctx, build)
continue
}
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
// landing then is.
if m.opts.Claim != nil && !m.opts.Claim() {
_ = msg.Nak()
continue
}
// A build outlives the acknowledgement window many times over; said while it runs,
// as the controller says it for its own long handlers, so the server neither hands
// the ask to a second machine nor counts the wait against its deliveries.
working := make(chan struct{})
go stillWorking(msg, working, heartbeatWhileWorking)
do(ctx, build)
close(working)
}
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
// landing then is.
if m.opts.Claim != nil && !m.opts.Claim() {
_ = msg.Nak()
continue
}
// A build outlives the acknowledgement window many times over; said while it runs,
// as the controller says it for its own long handlers, so the server neither hands
// the ask to a second machine nor counts the wait against its deliveries.
working := make(chan struct{})
go stillWorking(msg, working, heartbeatWhileWorking)
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
// deleted when it ends, whatever the outcome.
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
do(ctx, build)
stopRunning()
close(working)
}
}
// bind binds a queue's worker, which exists and is not created here.
//
// **The consumer's own filter, not the one subject this machine cares about.** The client checks what is asked for
// against the consumer's filter and refuses anything that is not the same — "subject does not match consumer" — so
// subscribing `…accept.build` against a consumer filtered on `…accept.>` is rejected even though it is narrower. Learned
// twice now, on two different consumers, which is why it is written down here.
func (m *natsMachine) bind(q *takeQueue) error {
sub, err := m.js.Context().PullSubscribe(q.worker.Filters[0], q.worker.Name,
nats.Bind(q.worker.Stream, q.worker.Name), nats.ManualAck())
if err != nil {
return err
}
q.sub, q.bound = sub, true
return nil
}
// next takes the next request in the order of the queues: each tried for a moment, and when all are empty the first waited
// on for a second. Nothing, with no error, when there was nothing.
func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
var last error
for _, q := range m.subs {
if !q.bound {
continue
}
trying, end := context.WithTimeout(ctx, checkTakeWait)
fetched, err := q.sub.Fetch(1, nats.Context(trying))
end()
if len(fetched) > 0 {
return fetched[0], q, nil
}
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) &&
!errors.Is(err, context.Canceled) {
last = err
}
if ctx.Err() != nil {
return nil, nil, nil
}
}
if last != nil {
return nil, nil, last
}
waiting, end := context.WithTimeout(ctx, takeIdleWait)
defer end()
fetched, err := m.subs[0].sub.Fetch(1, nats.Context(waiting))
if len(fetched) > 0 {
return fetched[0], m.subs[0], nil
}
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) &&
!errors.Is(err, context.Canceled) {
return nil, nil, err
}
return nil, nil, nil
}
// judgedAlready says why a check just taken from the other checks' queue need not run, or nothing: its pull request's key
// holds its head and trunk commit and a verdict. A key that cannot be read, or none, runs the check (the key is
// mesh-delivery's, and a check wrongly dropped is worse than one run twice).
func (m *natsMachine) judgedAlready(ctx context.Context, r BuildRequest) string {
c := r.Check
if c == nil || c.Try || (c.Group == "" && c.Number <= 0) {
return ""
}
if c.Group == "" {
key, found, err := ReadPullKey(ctx, m.js.Conn(), c.Owner, c.Repo, c.Number)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read the key of %s/%s#%d, so its check runs: %v\n", c.Owner, c.Repo, c.Number, err)
} else if found && key.Judged(r.Ref, c.Trunk) {
return fmt.Sprintf("%s/%s#%d is judged already at %.12s on %.12s (%s)", c.Owner, c.Repo, c.Number, r.Ref, c.Trunk,
key.Verdict)
}
}
return m.beingChecked(ctx, r)
}
// beingChecked says why a request need not run, or nothing: another builder's check of the same head and trunk commit is
// running, and says so under its pull request's key in the bucket of the checks running (ADR 0406 §3). A key a builder
// stopped renewing is gone within the bucket's age, and the request is taken then. Never the ask's own id: a redelivered ask
// is its own, and a key an older builder wrote that names no head or trunk is no match.
func (m *natsMachine) beingChecked(ctx context.Context, r BuildRequest) string {
running, ok, err := ReadRunning(ctx, m.js.Conn(), r)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read whether %s is being checked, so its check runs: %v\n", r.ID, err)
return ""
}
if ok && running.Build != r.ID && running.Same(r) {
return fmt.Sprintf("%s is being checked at %.12s on %.12s by %s as %s", r.ID, r.Ref, r.Check.Trunk, running.Builder,
running.Build)
}
return ""
}
type natsBuild struct {
@@ -398,6 +526,12 @@ func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error {
func (b *natsBuild) Done() error { return b.msg.Ack() }
// WatchPull has moved called once the pull request's key moves on from what this check was asked for; the stop ends the
// watch (see WatchPull). A request that is no pull request's check watches nothing.
func (b *natsBuild) WatchPull(ctx context.Context, moved func(PullKey)) (func(), error) {
return WatchPull(ctx, b.js.Conn(), b.request, moved)
}
// Began publishes that this machine has taken the build, into the stream like the outcome, so a
// reader that asks afterwards sees when it started as well as how it ended.
func (b *natsBuild) Began(ctx context.Context) error {
internal/link/builds_nats_test.go
+6
View File
@@ -44,6 +44,12 @@ func aBusWithTheBuildRole(t *testing.T) *broker.JetStream {
}); err != nil {
t.Fatal(err)
}
// And the two check queues' workers, which the controller makes beside the streams (novox/hq ADR 0406 §3).
for _, w := range broker.CheckWorkers() {
if err := js.EnsureConsumer(w); err != nil {
t.Fatal(err)
}
}
clean := func() {
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
for _, s := range broker.MeshStreams() {
internal/link/checks_nats.go
+293
View File
@@ -0,0 +1,293 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// A pull request's check is queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974).
// SupersededByKey is the outcome of a check that stopped because its pull request's key moved on: its head or its trunk
// commit is no longer the one it was asked for. Said as the check's failure, as a kill is; nothing is asked of the forge.
const SupersededByKey = "superseded: the pull request's key moved on"
// unsafeToken is what may not stand in a subject token or a key.
var unsafeToken = strings.NewReplacer(".", "_", "*", "_", ">", "_", " ", "_", "/", "_", "\t", "_", "\n", "_")
// tokenOf is a name as one subject token: any dot, wildcard or space replaced by an underscore.
func tokenOf(name string) string {
if name == "" {
return "_"
}
return unsafeToken.Replace(name)
}
// PullKeyOf is a pull request's key in mesh-delivery's bucket, and the last tokens of the subject its check is asked on:
// `<owner>.<repo>.<number>`, a dot in a name made an underscore.
func PullKeyOf(owner, repo string, number int) string {
return tokenOf(owner) + "." + tokenOf(repo) + "." + strconv.Itoa(number)
}
// CheckSubjectOf is the subject a check is asked on, and whether it has one of its own: a check of a pull request on
// the ready queue or the other, by the pull request; a delivery group's composed check by `group.<name>`; any other
// check (a branch tried before its pull request) by its own id, so it replaces nothing.
func CheckSubjectOf(seat string, r BuildRequest) (string, bool) {
c := r.Check
if seat != broker.TheBuildSeat || c == nil {
return "", false
}
prefix := broker.CheckSubjectPrefix(seat, c.Ready)
switch {
case c.Group != "":
return prefix + "group." + tokenOf(c.Group), true
case c.Try || c.Number <= 0:
return prefix + "try." + tokenOf(r.ID), true
}
return prefix + PullKeyOf(c.Owner, c.Repo, c.Number), true
}
// WorkSubjectOf is where an ask is put: a check on the build seat on its own queue, anything else on the seat's.
func WorkSubjectOf(seat string, r BuildRequest) string {
if subject, ok := CheckSubjectOf(seat, r); ok {
return subject
}
return BuildWorkOf(seat)
}
// PullKey is what mesh-delivery keeps of a pull request (ADR 0406 §4): its head, the trunk commit its current check is
// asked against, and its verdict once judged.
type PullKey struct {
Head string `json:"head"`
Trunk string `json:"trunk"`
Verdict string `json:"verdict,omitempty"`
}
// sameCommit says two spellings name one commit; either may be short, but not under twelve characters.
func sameCommit(a, b string) bool {
return a == b || SameHead(a, b)
}
// Moved says a key no longer holds what a check was asked for: its head or its trunk commit differs.
func (k PullKey) Moved(head, trunk string) bool {
return !sameCommit(k.Head, head) || (trunk != "" && k.Trunk != "" && !sameCommit(k.Trunk, trunk))
}
// Judged says a key holds the verdict of the check of this head against this trunk commit.
func (k PullKey) Judged(head, trunk string) bool {
// A verdict of a check that ran: a pass, a warning or a fail. An error is a check that could not run, and a request
// after it must run (issue 974, review 2).
judgedBy := k.Verdict == "pass" || k.Verdict == "warning" || k.Verdict == "fail"
return judgedBy && sameCommit(k.Head, head) && sameCommit(k.Trunk, trunk) && trunk != ""
}
// ReadPullKey reads a pull request's key from mesh-delivery's bucket: false when it has none.
func ReadPullKey(ctx context.Context, conn *nats.Conn, owner, repo string, number int) (PullKey, bool, error) {
api, err := jetstream.New(conn)
if err != nil {
return PullKey{}, false, err
}
kv, err := api.KeyValue(ctx, broker.PullsBucket)
if err != nil {
return PullKey{}, false, fmt.Errorf("the pull requests' keys: %w", err)
}
e, err := kv.Get(ctx, PullKeyOf(owner, repo, number))
if errors.Is(err, jetstream.ErrKeyNotFound) {
return PullKey{}, false, nil
}
if err != nil {
return PullKey{}, false, err
}
var key PullKey
if err := json.Unmarshal(e.Value(), &key); err != nil {
return PullKey{}, false, fmt.Errorf("the key of %s/%s#%d is not a pull request's: %w", owner, repo, number, err)
}
return key, true, nil
}
// WatchPull calls moved once when the pull request's key moves on from what this check was asked for — its head or its
// trunk commit differs — and not for a key that only gains its verdict. It hears updates, not the key as it stands: the
// key and the request are written for one announcement by two writers, so a key found older than the request is not
// a move. The check is then stopped by whoever asked for it, without asking the forge (novox/hq issue 726). The stop
// ends the watch.
func WatchPull(ctx context.Context, conn *nats.Conn, r BuildRequest, moved func(PullKey)) (stop func(), err error) {
c := r.Check
if c == nil || c.Group != "" || c.Try || c.Number <= 0 {
return func() {}, nil
}
api, err := jetstream.New(conn)
if err != nil {
return nil, err
}
kv, err := api.KeyValue(ctx, broker.PullsBucket)
if err != nil {
return nil, fmt.Errorf("the pull requests' keys: %w", err)
}
watching, end := context.WithCancel(ctx)
w, err := kv.Watch(watching, PullKeyOf(c.Owner, c.Repo, c.Number), jetstream.UpdatesOnly())
if err != nil {
end()
return nil, err
}
go func() {
defer w.Stop()
for {
select {
case <-watching.Done():
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
var key PullKey
if json.Unmarshal(e.Value(), &key) != nil {
continue
}
if key.Moved(r.Ref, c.Trunk) {
moved(key)
return
}
}
}
}()
return end, nil
}
// checkTakeWait is how long a taking loop waits on one queue for an ask that is not there, and takeIdleWait how long it
// waits on the first when all three are empty.
const (
checkTakeWait = 30 * time.Millisecond
takeIdleWait = time.Second
)
// CheckRunning is what a builder says of the check it is running, in the bucket of the checks running (novox/hq ADR 0406
// §3): the head and trunk commit it checks, the builder and its build. The key is the pull request's (RunningKeyOf).
type CheckRunning struct {
Head string `json:"head"`
Trunk string `json:"trunk"`
Builder string `json:"builder"`
Build string `json:"build"`
}
// RunningKeyOf is the key a check is said running under: its pull request's tokens, or `group.<name>` for a group's
// composed check; none for a try, which no other request replaces.
func RunningKeyOf(r BuildRequest) (string, bool) {
c := r.Check
switch {
case c == nil, c.Try:
return "", false
case c.Group != "":
return "group." + tokenOf(c.Group), true
case c.Number <= 0:
return "", false
}
return PullKeyOf(c.Owner, c.Repo, c.Number), true
}
// runningBucket opens the bucket of the checks running.
func runningBucket(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
api, err := jetstream.New(conn)
if err != nil {
return nil, err
}
kv, err := api.KeyValue(ctx, broker.CheckRunningBucket)
if err != nil {
return nil, fmt.Errorf("the checks running: %w", err)
}
return kv, nil
}
// ReadRunning reads the key of the check running for a request's pull request: false when there is none.
func ReadRunning(ctx context.Context, conn *nats.Conn, r BuildRequest) (CheckRunning, bool, error) {
key, ok := RunningKeyOf(r)
if !ok {
return CheckRunning{}, false, nil
}
kv, err := runningBucket(ctx, conn)
if err != nil {
return CheckRunning{}, false, err
}
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return CheckRunning{}, false, nil
}
if err != nil {
return CheckRunning{}, false, err
}
var running CheckRunning
if err := json.Unmarshal(e.Value(), &running); err != nil {
return CheckRunning{}, false, nil // a key of a builder from before this means no match
}
return running, true, nil
}
// Same says a running check is for this request's head and trunk commit: both said, and the same. A key that names neither
// (an older builder's) is no match.
func (c CheckRunning) Same(r BuildRequest) bool {
return r.Check != nil && c.Head != "" && c.Trunk != "" && r.Check.Trunk != "" && sameCommit(c.Head, r.Ref) &&
sameCommit(c.Trunk, r.Check.Trunk)
}
// keepRunning says a check is running: its key put now, put again at each heartbeat, and deleted by the stop it returns,
// whatever the outcome. Said to nobody that cannot be told: a bucket that is not there is no reason not to run the check.
func keepRunning(ctx context.Context, conn *nats.Conn, r BuildRequest, builder string, every time.Duration) (stop func()) {
key, ok := RunningKeyOf(r)
if !ok {
return func() {}
}
kv, err := runningBucket(ctx, conn)
if err != nil {
fmt.Fprintf(os.Stderr, "%s runs without saying so: %v\n", r.ID, err)
return func() {}
}
trunk := ""
if r.Check != nil {
trunk = r.Check.Trunk
}
body, _ := json.Marshal(CheckRunning{Head: r.Ref, Trunk: trunk, Builder: builder, Build: r.ID})
put := func() {
putting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer end()
if _, err := kv.Put(putting, key, body); err != nil {
fmt.Fprintf(os.Stderr, "%s could not say it is running: %v\n", r.ID, err)
}
}
put()
done := make(chan struct{})
finished := make(chan struct{})
go func() {
defer close(finished)
tick := time.NewTicker(every)
defer tick.Stop()
for {
select {
case <-done:
return
case <-tick.C:
put()
}
}
}()
return func() {
close(done)
<-finished
deleting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer end()
if err := kv.Delete(deleting, key); err != nil {
fmt.Fprintf(os.Stderr, "%s could not take its running key away; it ages out: %v\n", r.ID, err)
}
}
}
internal/link/checks_nats_test.go
+650
View File
@@ -0,0 +1,650 @@
package link
import (
"context"
"encoding/json"
"reflect"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// The checks of pull requests are queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974).
func aPullCheck(id string, number int, head, trunk string, ready bool) BuildRequest {
return BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head,
Check: &CheckRequest{Owner: "novox", Repo: "a", Number: number, Trunk: trunk, Ready: ready}}
}
// The subject a check is asked on says its queue and its pull request: ready or other, `<owner>.<repo>.<number>` with
// a dot in a name made an underscore, `group.<name>` for a group's, the id for a try; a build is on the seat's own queue.
func TestACheckIsAskedOnItsQueueBySubject(t *testing.T) {
for _, c := range []struct {
r BuildRequest
want string
}{
{aPullCheck("b1", 7, "h", "t", false), "mesh.seat.node-build-agent.check.other.novox.a.7"},
{aPullCheck("b2", 7, "h", "t", true), "mesh.seat.node-build-agent.check.ready.novox.a.7"},
{BuildRequest{ID: "b3", Check: &CheckRequest{Owner: "no.vox", Repo: "a.b", Number: 12}}, "mesh.seat.node-build-agent.check.other.no_vox.a_b.12"},
{BuildRequest{ID: "b4", Check: &CheckRequest{Owner: "novox", Repo: "a", Group: "g.1", Ready: true}}, "mesh.seat.node-build-agent.check.ready.group.g_1"},
{BuildRequest{ID: "b5", Check: &CheckRequest{Owner: "novox", Repo: "a", Try: true}}, "mesh.seat.node-build-agent.check.other.try.b5"},
{BuildRequest{ID: "b6", Repository: "/r"}, "mesh.seat.node-build-agent.accept.build"},
{BuildRequest{ID: "b7", Repository: "/r", DryRun: true}, "mesh.seat.node-build-agent.accept.build"},
} {
if got := WorkSubjectOf(TheBuildMachine, c.r); got != c.want {
t.Errorf("%s is asked on %s; want %s", c.r.ID, got, c.want)
}
}
// The retired role has no check queues: its checks wait with its builds.
if got := WorkSubjectOf(TheBuildMachineBefore, aPullCheck("b8", 7, "h", "t", true)); got != "mesh.seat.mesh-build-machine.accept.build" {
t.Errorf("a check of the retired role is asked on %s", got)
}
}
func queueOf(t *testing.T, js *broker.JetStream) Queue {
t.Helper()
q, err := ReadQueue(context.Background(), js, TheBuildMachine)
if err != nil {
t.Fatal(err)
}
return q
}
// A newer request for a pull request replaces the one still waiting, by the stream's rule, and each open pull request is
// left with one queued check (acceptance 5).
func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false),
aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false),
aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk
aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false),
aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
var got []string
for _, a := range queueOf(t, js).Asks {
if a.State != AskWaiting {
t.Errorf("%s is %s", a.ID, a.State)
}
got = append(got, a.ID)
}
if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) {
t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want)
}
}
// A delivery build is taken before a check of a ready pull request, that before any other check, and the others in the
// order they were asked.
func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("other-1", 1, "h1", "t", false),
aPullCheck("other-2", 2, "h2", "t", false),
aPullCheck("ready-3", 3, "h3", "t", true),
{ID: "delivery-4", Repository: "/d"},
aPullCheck("other-5", 5, "h5", "t", false),
aPullCheck("ready-6", 6, "h6", "t", true),
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
took := make(chan string, 8)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
var order []string
for len(order) < 6 {
select {
case id := <-took:
order = append(order, id)
case <-time.After(15 * time.Second):
t.Fatalf("taken so far: %v", order)
}
}
if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) {
t.Fatalf("taken as %v; want %v", order, want)
}
}
// aKeyBucket makes mesh-delivery's bucket of pull requests' keys, which the bus here has not been given by that module.
func aKeyBucket(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
t.Helper()
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.PullsBucket})
if err != nil {
t.Fatal(err)
}
return kv
}
func putKey(t *testing.T, kv jetstream.KeyValue, number int, k PullKey) {
t.Helper()
body, _ := json.Marshal(k)
if _, err := kv.Put(t.Context(), PullKeyOf("novox", "a", number), body); err != nil {
t.Fatal(err)
}
}
// A request left on the third queue for a head and a trunk commit its key says were judged is acknowledged and dropped when a
// builder takes it; one for a head the key does not say judged is run.
func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aKeyBucket(t, js)
ctx, stop := context.WithCancel(t.Context())
defer stop()
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged
putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false),
aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false),
aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false),
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
took := make(chan string, 8)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
var ran []string
for len(ran) < 2 {
select {
case id := <-took:
ran = append(ran, id)
case <-time.After(15 * time.Second):
t.Fatalf("run so far: %v", ran)
}
}
if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) {
t.Fatalf("run %v; want %v, the judged one dropped", ran, want)
}
time.Sleep(300 * time.Millisecond)
if n := len(queueOf(t, js).Asks); n != 0 {
t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n)
}
}
// A running check stops when its key's head moves or its trunk commit does, not when the key only gains its verdict, and a
// group's check, which a member's merge in the group's order writes nothing to the key of, is not stopped at all.
func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aKeyBucket(t, js)
ctx, stop := context.WithCancel(t.Context())
defer stop()
moved := make(chan PullKey, 4)
req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false)
end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k })
if err != nil {
t.Fatal(err)
}
defer end()
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"})
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
select {
case k := <-moved:
t.Fatalf("stopped by a key that moved nothing: %+v", k)
case <-time.After(500 * time.Millisecond):
}
putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"})
select {
case k := <-moved:
if k.Head != "head-7-111111111" {
t.Errorf("stopped by %+v", k)
}
case <-time.After(5 * time.Second):
t.Fatal("a key whose head moved did not stop the check")
}
// And the trunk commit.
req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false)
end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k })
if err != nil {
t.Fatal(err)
}
defer end2()
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"})
select {
case <-moved:
case <-time.After(5 * time.Second):
t.Fatal("a key whose trunk commit moved did not stop the check")
}
// A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does
// have is the pull request's own.
group := BuildRequest{ID: "group-1", Ref: "head-7-000000000",
Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}}
end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k })
if err != nil {
t.Fatal(err)
}
defer end3()
putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"})
select {
case k := <-moved:
t.Fatalf("a group's check was stopped by %+v", k)
case <-time.After(500 * time.Millisecond):
}
}
func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) {
k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: "pass"}
if k.Moved("aaaaaaaaaaaa", "tttttttttttt") || !k.Moved("bbbbbbbbbbbbbbbb", "tttttttttttt") || !k.Moved("aaaaaaaaaaaa", "uuuuuuuuuuuuuu") {
t.Error("moved is wrong")
}
if !k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111") || k.Judged("aaaaaaaaaaaa1111", "") || (PullKey{Head: "a", Trunk: "t"}).Judged("a", "t") {
t.Error("judged is wrong")
}
}
// The queue is read across the three streams in the order they are handed out, each ask says the stream it waits in, and an
// ask is removed from that stream.
func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("other-1", 1, "h1", "t", false),
aPullCheck("ready-2", 2, "h2", "t", true),
{ID: "delivery-3", Repository: "/d"},
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
q := queueOf(t, js)
var ids, streams []string
for _, a := range q.Asks {
ids, streams = append(ids, a.ID), append(streams, a.Stream)
}
if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) {
t.Fatalf("read %v; want the order they are handed out, %v", ids, want)
}
if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) {
t.Fatalf("streams %v; want %v", streams, want)
}
ready, _ := q.Find("ready-2")
if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker {
t.Errorf("the ready check is handed out by %s", w.Name)
}
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq {
t.Fatalf("deleted %d, %v", removed, err)
}
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 {
t.Fatalf("deleted again: %d, %v", removed, err)
}
if n := len(queueOf(t, js).Asks); n != 2 {
t.Errorf("%d asks left; want 2", n)
}
}
// aRunningBucket makes the bucket the builders say the checks they run in, with an age of its own short enough for a test
// (the mesh's is two minutes): a key not renewed within it is gone.
func aRunningBucket(t *testing.T, js *broker.JetStream, age time.Duration) jetstream.KeyValue {
t.Helper()
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: age})
if err != nil {
t.Fatal(err)
}
return kv
}
// A builder taking a request for a pull request's head and trunk commit that another builder is running a check of drops it:
// the bucket of running checks names the head and the trunk, the builder and the build (issue 974).
func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) {
js := aBusWithTheBuildRole(t)
aKeyBucket(t, js)
aRunningBucket(t, js, 2*time.Minute)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
running := make(chan string, 4)
release := make(chan struct{})
a := MachineOverNATS(js, "builder-a")
defer a.Close()
go func() {
_ = a.Take(ctx, func(ctx context.Context, work Build) {
running <- work.Request().ID
<-release
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
_ = work.Done()
})
}()
if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
select {
case <-running:
case <-time.After(10 * time.Second):
t.Fatal("builder a took nothing")
}
// The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it.
if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
b := MachineOverNATS(js, "builder-b")
defer b.Close()
took := make(chan string, 4)
go func() {
_ = b.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
_ = work.Done()
})
}()
select {
case id := <-took:
t.Fatalf("builder b took %s, which builder a is running a check of", id)
case <-time.After(3 * time.Second):
}
if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n)
}
// Another trunk commit is another check.
if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
if id != "moved" {
t.Fatalf("builder b took %s", id)
}
case <-time.After(10 * time.Second):
t.Fatal("a check against another trunk commit was dropped too")
}
close(release)
}
// A key that is not renewed ages out, and the request for its head and trunk commit is then taken: a builder that died
// running a check leaves nothing that holds the next one back for longer than the bucket's age.
func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) {
js := aBusWithTheBuildRole(t)
aKeyBucket(t, js)
kv := aRunningBucket(t, js, 2*time.Second)
ctx, stop := context.WithCancel(t.Context())
defer stop()
body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"})
if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil {
t.Fatal(err)
}
asks := BuildsOn(js, TheBuildMachine)
took := make(chan string, 4)
b := MachineOverNATS(js, "builder-b")
defer b.Close()
go func() {
_ = b.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
_ = work.Done()
})
}()
if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
t.Fatalf("%s was taken while the key was fresh", id)
case <-time.After(1200 * time.Millisecond):
}
time.Sleep(2500 * time.Millisecond) // the key was not renewed
if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
if id != "after-it-aged" {
t.Fatalf("took %s", id)
}
case <-time.After(10 * time.Second):
t.Fatal("the request was not taken after the key aged out")
}
}
// A builder running a check renews its key on each heartbeat, so it outlives the bucket's age, and deletes it when the check
// ends, whatever the outcome.
func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aRunningBucket(t, js, 2*time.Second)
was := heartbeatWhileWorking
heartbeatWhileWorking = 400 * time.Millisecond
t.Cleanup(func() { heartbeatWhileWorking = was })
ctx, stop := context.WithCancel(t.Context())
defer stop()
running := make(chan struct{}, 1)
release := make(chan struct{})
m := MachineOverNATS(js, "builder-a")
defer m.Close()
go func() {
_ = m.Take(ctx, func(ctx context.Context, work Build) {
running <- struct{}{}
<-release
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
_ = work.Done()
})
}()
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
<-running
key := PullKeyOf("novox", "a", 7)
e, err := kv.Get(ctx, key)
if err != nil {
t.Fatalf("no key while the check runs: %v", err)
}
var got CheckRunning
_ = json.Unmarshal(e.Value(), &got)
if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" {
t.Errorf("the key says %+v", got)
}
time.Sleep(3 * time.Second) // longer than the bucket's age
if _, err := kv.Get(ctx, key); err != nil {
t.Fatalf("the key aged out under a check still running: %v", err)
}
close(release)
for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) {
if _, err := kv.Get(ctx, key); err != nil {
break
}
if time.Now().After(end) {
t.Fatal("the key stays after the check ended")
}
}
}
// A key whose verdict is "error" — a check that could not run, or was stopped — judged nothing: a request for that head and
// trunk commit must run (issue 974, review 2).
func TestAnErrorVerdictJudgesNothing(t *testing.T) {
for verdict, judged := range map[string]bool{"pass": true, "warning": true, "fail": true, "error": false, "": false, "pending": false} {
k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: verdict}
if got := k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111"); got != judged {
t.Errorf("a key with verdict %q judged = %v; want %v", verdict, got, judged)
}
}
}
// A check stopped because the trunk moved leaves the newer request, for the head against the new trunk commit, taken and
// run: the stop is the stopped one's failure, not the key's verdict, and the newer request is not dropped by it.
func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aKeyBucket(t, js)
aRunningBucket(t, js, 2*time.Minute)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
outcomes := make(chan BuildResult, 8)
watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) {
var r BuildResult
if json.Unmarshal(m.Data, &r) == nil {
outcomes <- r
}
})
if err != nil {
t.Fatal(err)
}
defer func() { _ = watching.Unsubscribe() }()
ran := make(chan string, 4)
m := MachineOverNATS(js, "builder-a")
defer m.Close()
go func() {
_ = m.Take(ctx, func(ctx context.Context, work Build) {
building, cancel := context.WithCancel(ctx)
defer cancel()
ran <- work.Request().ID
stopped := make(chan struct{}, 1)
end, err := work.(interface {
WatchPull(context.Context, func(PullKey)) (func(), error)
}).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() })
if err != nil {
t.Error(err)
return
}
defer end()
if work.Request().ID == "older" {
select {
case <-stopped:
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey})
case <-time.After(15 * time.Second):
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"})
}
} else {
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}})
}
_ = work.Done()
})
}()
if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
if id := <-ran; id != "older" {
t.Fatalf("took %s", id)
}
time.Sleep(300 * time.Millisecond) // the watch is made
// The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked.
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"})
if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-ran:
if id != "newer" {
t.Fatalf("took %s", id)
}
case <-time.After(15 * time.Second):
t.Fatal("the newer request, for the same head on the new trunk commit, was not taken")
}
got := map[string]string{}
for len(got) < 2 {
select {
case r := <-outcomes:
got[r.ID] = r.Failed
case <-time.After(10 * time.Second):
t.Fatalf("outcomes so far: %v", got)
}
}
if got["older"] != SupersededByKey || got["newer"] != "" {
t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got)
}
}
// An unacknowledged check goes to a second builder: taken from a check worker and not acknowledged, it is handed out again
// when the worker's wait is up.
func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
for _, ready := range []bool{false, true} {
js := aBusWithTheBuildRole(t)
name, worker := "other", broker.ChecksOtherWorker
if ready {
name, worker = "ready", broker.ChecksReadyWorker
}
// A worker whose wait is a second, as the mesh's is a minute.
for _, w := range broker.CheckWorkers() {
if w.Name == worker {
w.AckWaitSeconds = 1
if err := js.EnsureConsumer(w); err != nil {
t.Fatal(err)
}
}
}
ctx, stop := context.WithCancel(t.Context())
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil {
t.Fatal(err)
}
// The first builder takes it and dies: its pull is fetched, never acknowledged.
if _, err := fetchOne(js, worker); err != nil {
t.Fatal(err)
}
took := make(chan BuildRequest, 2)
second := MachineOverNATS(js, "builder-b")
go func() {
_ = second.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request()
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
_ = work.Done()
})
}()
select {
case r := <-took:
if r.ID != "dies-"+name {
t.Errorf("%s: took %s", name, r.ID)
}
case <-time.After(15 * time.Second):
t.Errorf("%s: the unacknowledged check did not go to a second builder", name)
}
second.Close()
stop()
}
}
// fetchOne takes the next message of a worker and leaves it unacknowledged, as a builder that died would.
func fetchOne(js *broker.JetStream, worker string) (*nats.Msg, error) {
for _, w := range broker.CheckWorkers() {
if w.Name != worker {
continue
}
sub, err := js.Context().PullSubscribe(w.Filters[0], w.Name, nats.Bind(w.Stream, w.Name), nats.ManualAck())
if err != nil {
return nil, err
}
got, err := sub.Fetch(1, nats.MaxWait(5*time.Second))
if err != nil {
return nil, err
}
return got[0], nil
}
return nil, nil
}
internal/link/events.go
+7
View File
@@ -76,6 +76,10 @@ const (
// KeyChecked: a pull request's merge check was judged (novox/hq to-be 45 §9) — the verdict, its
// summary and its report, for the forge's holder to set as the pull request's status.
KeyChecked = "checked"
// KeySent: a plan's tier was sent to a machine, and the send kept (novox/hq ADR 0406 §5, issue 974): the plan, the tier,
// the machine, the modules and when. What an outsider hears of a rollout; the controller's own next step does not
// wait for it.
KeySent = "sent"
// KeyPlanMoved: a walk — the controller's plan of one trunk commit, tier by tier across the machines —
// was kept in a new state (novox/hq ADR 0239): the plan whole, for the delivery it walks to read its
// steps from. Said after every save of the serving controller; a delivery's holder also asks for the
@@ -260,6 +264,9 @@ type PullUpdated struct {
// check's second layer; MergeCheckSaid says the announcer looked.
MergeCheck bool `json:"merge_check,omitempty"`
MergeCheckSaid bool `json:"merge_check_said,omitempty"`
// Ready says the pull request is marked ready (novox/hq ADR 0406 §3): not the forge's word, mesh-delivery's, said
// to the controller with the head to check again; its check is queued before every other check.
Ready bool `json:"ready,omitempty"`
}
// Checked is a pull request's merge check, judged: what the controller says as `checked`.
internal/link/queue.go
+99 -49
View File
@@ -62,11 +62,12 @@ type QueuedAsk struct {
Path string `json:"path,omitempty"`
Ref string `json:"ref,omitempty"`
AskedAt time.Time `json:"asked-at,omitempty"`
// Stored is when the stream first took it, a move into order kept (novox/hq issue 558).
// Stored is when the stream took it.
Stored time.Time `json:"stored,omitempty"`
State string `json:"state"`
// movedFrom is the MovedFromHeader it carries: the sequence of the message a move copied it from.
movedFrom string
// Stream is the work queue holding it: the seat's own, or one of the two check queues (novox/hq ADR 0406 §3). Its
// Seq is the sequence there.
Stream string `json:"stream,omitempty"`
// On and Started are the holder building an in-flight ask and when it started, from its started
// event. Was is the holder that started an in-flight ask and is no longer building it — handed
// back, to be handed out again — with Started its start there.
@@ -201,62 +202,113 @@ func startedAt(s BuildStart) time.Time {
return t
}
// ReadQueue reads a build seat's work queue: the stream's asks, the worker's position, and what the
// seat's events said about the builds behind it. Through the JetStream API alone (STREAM.INFO,
// CONSUMER.INFO, STREAM.MSG.GET), which only the controller's account reaches.
func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) {
// queueStream is one stream a seat's asks wait in, and the worker that hands them out.
type queueStream struct {
Worker broker.Consumer
Filter string
}
// queueStreamsOf are the streams a seat's asks wait in, in the order a builder takes from them (novox/hq ADR 0406 §3): the
// seat's own queue, where a delivery build waits, then the checks of pull requests marked ready, then every other check.
// Only the current build role has the check queues; the retired one's checks wait on its own.
func queueStreamsOf(seat string) []queueStream {
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
streams := []queueStream{{Worker: worker, Filter: BuildWorkOf(seat)}}
if seat == broker.TheBuildSeat {
for _, w := range broker.CheckWorkers() {
streams = append(streams, queueStream{Worker: w, Filter: w.Filters[0]})
}
}
return streams
}
// WorkerOf is the worker that hands out an ask, by the queue it waits in.
func WorkerOf(seat string, ask QueuedAsk) broker.Consumer {
streams := queueStreamsOf(seat)
for _, qs := range streams {
if qs.Worker.Stream == ask.Stream {
return qs.Worker
}
}
return streams[0].Worker
}
// ReadQueue reads a build seat's work queues: the streams' asks, the workers' positions, and what the seat's events
// said about the builds behind them. Through the JetStream API alone (STREAM.INFO, CONSUMER.INFO, STREAM.MSG.GET),
// which only the controller's account reaches. The asks come in the order they are handed out: the seat's own queue's,
// the ready checks', the other checks'.
func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) {
q := Queue{Seat: seat}
api, err := jetstream.New(js.Conn())
if err != nil {
return q, err
}
stream, err := api.Stream(ctx, worker.Stream)
if err != nil {
return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
type read struct {
asks []QueuedAsk
delivered uint64
ackPending int
}
consumer, err := stream.Consumer(ctx, worker.Name)
switch {
case errors.Is(err, jetstream.ErrConsumerNotFound):
// No holder has ever been given a worker: everything in the stream is waiting.
case err != nil:
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
default:
info, err := consumer.Info(ctx)
var reads []read
for i, qs := range queueStreamsOf(seat) {
stream, err := api.Stream(ctx, qs.Worker.Stream)
if err != nil {
return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, qs.Worker.Stream, err)
}
var r read
consumer, err := stream.Consumer(ctx, qs.Worker.Name)
switch {
case errors.Is(err, jetstream.ErrConsumerNotFound):
// No holder has ever been given a worker: everything in the stream is waiting.
case err != nil:
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
default:
info, err := consumer.Info(ctx)
if err != nil {
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
}
r.delivered, r.ackPending = info.Delivered.Stream, info.NumAckPending
if i == 0 {
q.Delivered, q.MaxDeliver = info.Delivered.Stream, info.Config.MaxDeliver
}
q.AckPending += info.NumAckPending
if q.MaxDeliver == 0 {
q.MaxDeliver = info.Config.MaxDeliver
}
}
q.Delivered = info.Delivered.Stream
q.AckPending = info.NumAckPending
q.MaxDeliver = info.Config.MaxDeliver
// Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this
// is a handful of reads, never the history.
var seq uint64 = 1
for n := 0; n < 10000; n++ {
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(qs.Filter))
if errors.Is(err, jetstream.ErrMsgNotFound) {
break
}
if err != nil {
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
}
a := askOf(msg)
a.Stream = qs.Worker.Stream
r.asks = append(r.asks, a)
seq = msg.Sequence + 1
}
reads = append(reads, r)
}
// Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this
// is a handful of reads, never the history.
var asks []QueuedAsk
var seq uint64 = 1
for n := 0; n < 10000; n++ {
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
if errors.Is(err, jetstream.ErrMsgNotFound) {
break
}
if err != nil {
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
}
asks = append(asks, askOf(msg))
seq = msg.Sequence + 1
}
// What the events say, from shortly before the oldest ask behind the worker: its start, if any,
// came after it was asked.
// What the events say, from shortly before the oldest ask behind a worker: its start, if any, came after it was asked.
var since time.Time
for _, a := range asks {
if a.Seq <= q.Delivered && (since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since))) {
since = a.AskedAt
behind := false
for _, r := range reads {
for _, a := range r.asks {
if a.Seq <= r.delivered {
behind = true
if since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since)) {
since = a.AskedAt
}
}
}
}
heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}}
if len(asks) > 0 && q.Delivered > 0 {
if behind {
if since.IsZero() {
since = time.Now().Add(-7 * 24 * time.Hour)
}
@@ -264,17 +316,15 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
return q, err
}
}
q.Asks = ClassifyQueue(asks, q.Delivered, q.AckPending, heard)
for _, r := range reads {
q.Asks = append(q.Asks, ClassifyQueue(r.asks, r.delivered, r.ackPending, heard)...)
}
return q, nil
}
// askOf is one ask as the stream holds it: what its request says, and when the stream took it.
func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
ask := QueuedAsk{Seq: msg.Sequence, Stored: msg.Time}
// Moved into order, it keeps when the stream first took it (novox/hq issue 558).
if at, err := time.Parse(time.RFC3339Nano, msg.Header.Get(QueuedAtHeader)); err == nil {
ask.Stored = at
}
if json.Unmarshal(msg.Data, &ask.Request) == nil {
r := ask.Request
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
@@ -282,7 +332,7 @@ func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
ask.AskedAt = at
}
} else {
// Nothing read is nothing kept: a half-read request would pass for a delivery (novox/hq issue 558).
// Nothing read is nothing kept: a half-read request would pass for a delivery.
ask.Request = BuildRequest{}
}
return ask
internal/link/queue_kind.go
+50
View File
@@ -0,0 +1,50 @@
package link
import (
"context"
"errors"
"strings"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// The kinds of ask, as the queue says them. **The kind decides the queue** (novox/hq ADR 0406 §3): a delivery build is on
// the seat's own queue, taken before any check; a check of a pull request marked ready is on the second; every other
// check on the third. Nothing orders the asks any more: a builder takes from the queues in that order, and the check
// queues keep one request a pull request. The reordering of one queue by moving asks to its tail, with its pause of the
// worker, its look every two seconds and its second copies, is gone (issue 558 is answered by the streams).
const (
KindDelivery = "delivery"
KindCheck = "check"
)
// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included —
// a dry run, or an ask nothing could be read from; a delivery for any other build.
func KindOf(r BuildRequest) string {
if r.ID == "" || r.Check != nil || r.DryRun {
return KindCheck
}
return KindDelivery
}
// DeleteAsk removes a waiting or dead ask from the queue it is in, at its sequence. Says the sequence it removed, 0 when
// it was not there any more: taken and settled, or replaced by a newer request for its pull request.
func DeleteAsk(_ context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) {
stream := ask.Stream
if stream == "" {
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
stream = worker.Stream
}
err := js.Context().DeleteMsg(stream, ask.Seq)
switch {
case err == nil:
return ask.Seq, nil
case errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) ||
strings.Contains(err.Error(), "no message found"):
return 0, nil
}
return 0, err
}
internal/link/queue_order.go
-433
View File
@@ -1,433 +0,0 @@
package link
import (
"context"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// The order the build queue hands asks out in (novox/hq issue 558).
//
// **A build a walk waits for goes before a pull request's check.** The queue was first come, first served,
// and a merged change's delivery build waited minutes behind checks while its walk, and every walk after
// it, waited on the build. So:
//
// - a delivery is handed out before any check;
// - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check
// waits for ever behind a stream of deliveries;
// - within a kind, first come, first served, by when it was asked;
// - a branch tried before its pull request goes after everything else, however long it has waited: it is
// never promoted ahead of a delivery, and never ahead of a pull request's check (novox/hq issue 686);
// - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check
// taken late breaks nothing.
//
// **Kept in the stream, never beside it.** The queue is the seat's work queue, which the server hands out
// in stream order to whichever holder pulls; nothing else holds the order, so a restarted controller
// finds it as it was left. An ask that must go behind another is moved to the tail — deleted from where
// it was and published again, its body unchanged, under the same id — while the worker is paused, so no
// holder takes it in the moment it moves. Only waiting asks move: what a holder took is never touched.
// The kinds of ask, as the queue says them.
const (
KindDelivery = "delivery"
KindCheck = "check"
)
// CheckWaitsAtMost is how long a check waits before it goes ahead of the deliveries asked after it.
var CheckWaitsAtMost = 10 * time.Minute
// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included —
// a dry run, or an ask nothing could be read from; a delivery for any other build.
func KindOf(r BuildRequest) string {
if r.ID == "" || r.Check != nil || r.DryRun {
return KindCheck
}
return KindDelivery
}
// askedOrStored is when an ask was asked, as its id says, or else when the queue took it.
func (a QueuedAsk) askedOrStored() time.Time {
if !a.AskedAt.IsZero() {
return a.AskedAt
}
return a.Stored
}
// QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check
// that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked.
func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk {
var first, then, tries []QueuedAsk
for _, a := range waiting {
if IsTry(a.Request) {
tries = append(tries, a)
continue
}
if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost {
then = append(then, a)
continue
}
first = append(first, a)
}
byAsking := func(s []QueuedAsk) {
sort.SliceStable(s, func(i, j int) bool {
if ti, tj := s[i].askedOrStored(), s[j].askedOrStored(); !ti.Equal(tj) {
return ti.Before(tj)
}
return s[i].Seq < s[j].Seq
})
}
byAsking(first)
byAsking(then)
byAsking(tries)
return append(append(first, then...), tries...)
}
// toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything
// after the longest start of order that the queue already holds in that order.
func toMove(queue, order []QueuedAsk) []QueuedAsk {
at := make(map[uint64]int, len(queue))
for i, a := range queue {
at[a.Seq] = i
}
kept, last := 0, -1
for kept < len(order) {
i, ok := at[order[kept].Seq]
if !ok || i < last {
break
}
last = i
kept++
}
return order[kept:]
}
// orderPasses keeps one pass at a time. Only the acting controller's loop runs one (KeepingInOrder), so
// there is one; the lock holds a test's two at once to that too.
var orderPasses sync.Mutex
// pausedFor bounds a pause the pass could not end itself: a controller that stops mid-pass leaves the
// worker paused this long at most. moveMargin is the part of it no move starts in: a pass stops moving
// with at least this left, and the next pass moves the rest, so no move runs past the pause.
const (
pausedFor = 20 * time.Second
moveMargin = 5 * time.Second
publishWaits = 3 * time.Second
)
// QueuedAtHeader carries, on a moved ask, when the stream first took it: a move keeps an ask's place in
// time, its id unread or not.
const QueuedAtHeader = "Mesh-Queued-At"
// MovedFromHeader names, on a move's copy, the sequence of the message it is a copy of: with QueuedAtHeader,
// what proves a second message of one id is a move's copy and not an ask delivered again (ADR 0293), whose
// headers are its original's — and so never name that original's own sequence.
const MovedFromHeader = "Mesh-Moved-From"
// Pass is what one pass did: the asks it moved, and each second copy it removed, in words.
type Pass struct {
Moved []string
Removed []string
}
// asked is the signal an ask sends the acting controller's pass loop, in this process. An ask asked in
// another process — a verb run as its own command — is seen by the loop as the queue's last sequence moving.
var asked = make(chan struct{}, 1)
// signalAsked tells the pass loop an ask joined the queue. Never blocks, never runs a pass.
func signalAsked() {
select {
case asked <- struct{}{}:
default:
}
}
// PutInOrder moves the seat's waiting asks into the order QueueOrder says, and says each it moved. A queue
// already in order is read and left alone; the worker is paused only while asks move. Run by the acting
// controller alone (KeepingInOrder): two passes in two processes would each resume what the other paused.
func PutInOrder(ctx context.Context, js *broker.JetStream, seat string) (Pass, error) {
var pass Pass
orderPasses.Lock()
defer orderPasses.Unlock()
api, err := jetstream.New(js.Conn())
if err != nil {
return pass, err
}
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
waiting, extra, delivered, err := readWaiting(ctx, api, worker, seat)
if err != nil || (len(extra) == 0 && len(toMove(waiting, QueueOrder(waiting, time.Now()))) == 0) {
return pass, err
}
// Paused, then read again: what the worker handed out up to the pause is not waiting any more.
pauseEnds := time.Now().Add(pausedFor)
if delivered >= 0 {
if _, err := api.PauseConsumer(ctx, worker.Stream, worker.Name, pauseEnds); err != nil {
return pass, fmt.Errorf("the %s worker could not be paused to put its queue in order, so it is left "+
"first come, first served: %w", seat, err)
}
defer func() {
resuming, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
if _, err := api.ResumeConsumer(resuming, worker.Stream, worker.Name); err != nil {
fmt.Printf("the %s worker could not be resumed after its queue was put in order; it resumes by "+
"itself by %s: %v\n", seat, pauseEnds.Format(time.RFC3339), err)
}
}()
if waiting, extra, _, err = readWaiting(ctx, api, worker, seat); err != nil {
return pass, err
}
}
stream, err := api.Stream(ctx, worker.Stream)
if err != nil {
return pass, err
}
// A second copy of one ask — a move that published and stopped before it deleted — goes first.
for _, a := range extra {
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
return pass, fmt.Errorf("a second copy of %s (seq %d) could not be removed: %w", a.ID, a.Seq, err)
}
pass.Removed = append(pass.Removed, fmt.Sprintf("%s at seq %d, a move's second copy of the message at seq %s",
a.ID, a.Seq, a.movedFrom))
}
for _, a := range toMove(waiting, QueueOrder(waiting, time.Now())) {
if delivered >= 0 && time.Until(pauseEnds) < moveMargin {
// The rest waits for the next pass, under a pause of its own.
break
}
ok, err := moveToTheTail(ctx, js, stream, worker.Name, seat, a)
if err != nil {
return pass, err
}
if ok {
pass.Moved = append(pass.Moved, a.ID)
}
}
return pass, nil
}
// readWaiting is every ask the worker has not yet handed out, in queue order, but each second copy of one
// ask; the second copies, to remove; and the last sequence the worker handed out, -1 when there is no
// worker, so nothing is handed out to pause.
//
// **A second copy is proven, never guessed from an id**: a waiting message whose MovedFromHeader names an
// earlier message of the same request id still in the queue, and whose QueuedAtHeader is that message's
// first-queued time — a move that published its copy and stopped before it deleted the original. The
// original stays, waiting or handed out (a holder's then, and the copy would build it twice). Any other
// second message of one id — an ask a dead letter delivered again beside its original (ADR 0293) — is a
// real ask and stays.
func readWaiting(ctx context.Context, api jetstream.JetStream, worker broker.Consumer, seat string) (
waiting, extra []QueuedAsk, delivered int64, err error) {
stream, err := api.Stream(ctx, worker.Stream)
if err != nil {
return nil, nil, 0, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
}
delivered = -1
if consumer, err := stream.Consumer(ctx, worker.Name); err == nil {
info, err := consumer.Info(ctx)
if err != nil {
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
}
delivered = int64(info.Delivered.Stream)
} else if !errors.Is(err, jetstream.ErrConsumerNotFound) {
return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
}
// From the first: the stream holds only what is unsettled, so this is a handful of reads.
// Every message by sequence, with its id and first-queued time, to prove a copy against.
type held struct {
id string
stored time.Time
}
bySeq := map[string]held{}
var seq uint64 = 1
for n := 0; n < 10000; n++ {
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
if errors.Is(err, jetstream.ErrMsgNotFound) {
break
}
if err != nil {
return nil, nil, 0, fmt.Errorf("reading the %s work queue: %w", seat, err)
}
seq = msg.Sequence + 1
a := askOf(msg)
a.movedFrom = msg.Header.Get(MovedFromHeader)
from, known := bySeq[a.movedFrom]
again := a.ID != "" && a.movedFrom != "" && known && from.id == a.ID && from.stored.Equal(a.Stored)
bySeq[strconv.FormatUint(a.Seq, 10)] = held{id: a.ID, stored: a.Stored}
switch {
case int64(a.Seq) <= delivered:
case again:
extra = append(extra, a)
default:
waiting = append(waiting, a)
}
}
return waiting, extra, delivered, nil
}
// moveToTheTail publishes the ask's body again at the queue's tail and then deletes it where it was. **The
// copy first**: a publish that fails deletes nothing, and a pass that stops between the two leaves two
// copies, which the next pass sees by their one id and makes one (readWaiting). A move never loses an ask.
// Looked at once more just before: one the worker has handed out since is a holder's now and stays. False
// when it was not moved.
func moveToTheTail(ctx context.Context, js *broker.JetStream, stream jetstream.Stream, workerName, seat string,
a QueuedAsk) (bool, error) {
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
info, err := consumer.Info(ctx)
if err != nil {
return false, err
}
if info.Delivered.Stream >= a.Seq {
return false, nil
}
}
msg, err := stream.GetMsg(ctx, a.Seq)
if errors.Is(err, jetstream.ErrMsgNotFound) {
return false, nil
}
if err != nil {
return false, err
}
again := &nats.Msg{Subject: msg.Subject, Data: msg.Data, Header: nats.Header{}}
for k, v := range msg.Header {
// The server's own headers stay behind: a message id would be taken for a duplicate and dropped.
if !strings.HasPrefix(k, "Nats-") {
again.Header[k] = v
}
}
// Its place in time goes with it: when the stream first took it, as askOf read it; and the message it
// is a copy of, which proves it one.
again.Header.Set(QueuedAtHeader, askOf(msg).Stored.UTC().Format(time.RFC3339Nano))
again.Header.Set(MovedFromHeader, strconv.FormatUint(a.Seq, 10))
// Inside the margin, so the publish ends while the worker is still paused.
publishing, cancel := context.WithTimeout(ctx, publishWaits)
ack, err := js.Context().PublishMsg(again, nats.Context(publishing))
cancel()
if err != nil {
return false, fmt.Errorf("%s (seq %d) could not be put behind, and stays where it is: %w", a.ID, a.Seq, err)
}
// **Handed out meanwhile** — a bus slow enough that the pause ended: the original is a holder's, and the
// copy goes instead, so it is built once.
if consumer, err := stream.Consumer(ctx, workerName); err == nil {
info, err := consumer.Info(ctx)
if err != nil || info.Delivered.Stream >= a.Seq {
if derr := stream.DeleteMsg(ctx, ack.Sequence); derr != nil && !errors.Is(derr, jetstream.ErrMsgNotFound) {
return false, fmt.Errorf("%s was handed out while it moved and its copy (seq %d) could not be "+
"removed; the next pass removes it: %w", a.ID, ack.Sequence, derr)
}
if err != nil {
return false, fmt.Errorf("whether %s was handed out while it moved cannot be read, so its copy was "+
"removed and it stays where it was: %w", a.ID, err)
}
if info.Delivered.Stream >= ack.Sequence {
return false, fmt.Errorf("%s was handed out while it moved, and its copy too: it may be built twice", a.ID)
}
return false, nil
}
}
if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
return false, fmt.Errorf("%s was copied behind and its original (seq %d) not removed; the next pass "+
"removes the second copy: %w", a.ID, a.Seq, err)
}
return true, nil
}
// KeepingInOrder runs the passes, on the acting controller alone, until ctx ends: when an ask in this
// process says so, when the queue's last sequence moved — an ask from another process — and every
// passEvery, for a check that comes of age with nothing asked. seat says the build seat to order; acting
// whether this controller acts (Holding); said what a pass did.
func KeepingInOrder(ctx context.Context, js *broker.JetStream, seat func() string, acting func() bool,
said func(format string, args ...any)) {
tick := time.NewTicker(orderLooks)
defer tick.Stop()
var lastSeq uint64
var lastPass time.Time
for {
signalled := false
select {
case <-ctx.Done():
return
case <-asked:
signalled = true
case <-tick.C:
}
if !acting() {
continue
}
name := seat()
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: name, Accepts: []string{"build"}})
info, err := js.Context().StreamInfo(worker.Stream)
if err != nil {
continue
}
if !signalled && info.State.LastSeq == lastSeq && time.Since(lastPass) < passEvery {
continue
}
lastSeq, lastPass = info.State.LastSeq, time.Now()
passing, cancel := context.WithTimeout(ctx, pausedFor+10*time.Second)
pass, err := PutInOrder(passing, js, name)
cancel()
for _, r := range pass.Removed {
said("the %s queue: removed %s", name, r)
}
if len(pass.Moved) > 0 {
said("the %s queue put in order: %s moved behind what goes first", name, strings.Join(pass.Moved, ", "))
}
if err != nil {
said("the %s queue could not be put in order: %v", name, err)
}
if info, err := js.Context().StreamInfo(worker.Stream); err == nil {
// Its own moves are not an ask.
lastSeq = info.State.LastSeq
}
}
}
// orderLooks is how often the loop looks whether an ask joined from another process; passEvery how often
// it passes with nothing asked.
var (
orderLooks = 2 * time.Second
passEvery = 30 * time.Second
)
// DeleteAsk removes a waiting or dead ask from the queue: at its sequence, or — moved into order since it
// was read — the copy carrying its id. Says the sequence it removed, 0 when neither was there any more.
func DeleteAsk(ctx context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) {
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
gone := func(err error) bool {
return errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) ||
(err != nil && strings.Contains(err.Error(), "no message found"))
}
err := js.Context().DeleteMsg(worker.Stream, ask.Seq)
if err == nil {
return ask.Seq, nil
}
if !gone(err) {
return 0, err
}
if ask.ID == "" {
return 0, nil
}
q, err := ReadQueue(ctx, js, seat)
if err != nil {
return 0, err
}
for _, a := range q.Asks {
if a.ID == ask.ID && a.Seq != ask.Seq && a.State != AskInFlight {
if err := js.Context().DeleteMsg(worker.Stream, a.Seq); err != nil && !gone(err) {
return 0, err
} else if err == nil {
return a.Seq, nil
}
}
}
return 0, nil
}
internal/link/queue_order_test.go
-441
View File
@@ -1,441 +0,0 @@
package link
import (
"context"
"encoding/json"
"fmt"
"reflect"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any
// pull request's check, a check that has waited ten minutes before every delivery asked after it, and
// first come, first served within a kind.
func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk {
r := BuildRequest{ID: id, Repository: "/r"}
if check {
r.Check = &CheckRequest{Owner: "novox", Repo: "r"}
}
return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting}
}
func idsOf(asks []QueuedAsk) []string {
out := []string{}
for _, a := range asks {
out = append(out, a.ID)
}
return out
}
func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) {
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
waiting := []QueuedAsk{
anAsk(1, "check-a", now.Add(-3*time.Minute), true),
anAsk(2, "check-b", now.Add(-2*time.Minute), true),
anAsk(3, "delivery-d", now.Add(-time.Minute), false),
}
want := []string{"delivery-d", "check-a", "check-b"}
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
t.Fatalf("handed out as %v, want %v", got, want)
}
// Only the checks ahead of it move behind it; the delivery stays where it is.
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) {
t.Fatalf("moved %v", got)
}
}
func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) {
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
// As the queue stands after a delivery put the young checks behind it, ten minutes ago and since.
waiting := []QueuedAsk{
anAsk(5, "delivery-early", now.Add(-12*time.Minute), false),
anAsk(6, "delivery-later", now.Add(-time.Minute), false),
anAsk(7, "check-old", now.Add(-11*time.Minute), true),
anAsk(8, "check-young", now.Add(-9*time.Minute), true),
}
want := []string{"delivery-early", "check-old", "delivery-later", "check-young"}
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
t.Fatalf("handed out as %v, want %v", got, want)
}
if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) {
t.Fatalf("moved %v", got)
}
}
func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) {
for _, r := range []BuildRequest{
{}, // unreadable: nothing was read from it
{ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked
{ID: "build-2", DryRun: true}, // looked at, never delivered
} {
if got := KindOf(r); got != KindCheck {
t.Errorf("%+v is a %s", r, got)
}
}
if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery {
t.Errorf("a build is a %s", got)
}
}
// Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery.
func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(context.Background())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
now := time.Now()
for i, r := range []BuildRequest{
{ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}},
{ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}},
{ID: NewBuildID(now), Repository: "/d"},
} {
if i == 2 {
// And one nobody can read, whose kind cannot be told: it waits with the checks.
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
t.Fatal(err)
}
}
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
// The acting controller's loop puts it in order, told by the asks.
go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true },
func(string, ...any) {})
time.Sleep(500 * time.Millisecond)
took := make(chan string, 4)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().Repository
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
var order []string
for len(order) < 3 {
select {
case r := <-took:
order = append(order, r)
case <-time.After(10 * time.Second):
t.Fatalf("taken so far: %v", order)
}
}
if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) {
t.Fatalf("taken as %v, want %v", order, want)
}
}
// --- the moves, against a real server ------------------------------------------------------------
func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) {
t.Helper()
body, _ := json.Marshal(r)
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
t.Fatal(err)
}
}
func aCheck(asked time.Time, repository string) BuildRequest {
return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}}
}
func aDelivery(asked time.Time, repository string) BuildRequest {
return BuildRequest{ID: NewBuildID(asked), Repository: repository}
}
// queued is the stream's asks, in queue order.
func queued(t *testing.T, js *broker.JetStream) []QueuedAsk {
t.Helper()
q, err := ReadQueue(context.Background(), js, TheBuildMachine)
if err != nil {
t.Fatal(err)
}
return q.Asks
}
// takeAll runs a holder that records how often it was handed each repository.
func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) {
t.Helper()
took := make(chan string, 100)
var mu sync.Mutex
counts := map[string]int{}
machine := MachineOverNATS(js, "anchor")
t.Cleanup(machine.Close)
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
mu.Lock()
counts[work.Request().Repository]++
mu.Unlock()
took <- work.Request().Repository
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
return took, func() map[string]int {
mu.Lock()
defer mu.Unlock()
out := map[string]int{}
for k, v := range counts {
out[k] = v
}
return out
}
}
// A move that published its copy and stopped before deleting the original leaves two copies of one ask;
// the next pass makes them one, and a copy of one a holder already took is removed, never built twice.
func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
now := time.Now()
check := aCheck(now.Add(-2*time.Minute), "/c")
publishAsk(t, js, check)
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 {
t.Fatalf("removed %v: %v", pass.Removed, err)
}
var got []string
for _, a := range queued(t, js) {
got = append(got, a.Repository)
}
if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) {
t.Fatalf("after the pass the queue holds %v, want %v", got, want)
}
// And one a holder took before the copy was published: the copy goes, the holder's stays.
taken := aCheck(now, "/taken")
publishAsk(t, js, taken)
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
// Take everything waiting up to and including /taken, and keep /taken unsettled.
for {
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
if err != nil || len(msgs) == 0 {
t.Fatalf("nothing to take: %v", err)
}
var r BuildRequest
_ = json.Unmarshal(msgs[0].Data, &r)
if r.Repository == "/taken" {
break
}
_ = msgs[0].Ack()
}
moveCopyOf(t, js, taken.ID)
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
t.Fatal(err)
}
n := 0
for _, a := range queued(t, js) {
if a.ID == taken.ID {
n++
}
}
if n != 1 {
t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n)
}
}
// moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete
// would leave it.
func moveCopyOf(t *testing.T, js *broker.JetStream, id string) {
t.Helper()
for _, a := range queued(t, js) {
if a.ID != id {
continue
}
body, _ := json.Marshal(a.Request)
msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}}
msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano))
msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq))
if _, err := js.Context().PublishMsg(msg); err != nil {
t.Fatal(err)
}
return
}
t.Fatalf("%s is not queued", id)
}
// A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real
// ask with the same id, and is never removed as a second copy — even when the original was itself a move's
// copy, whose headers the delivery carries.
func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
ask := aCheck(time.Now(), "/again")
publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before"))
publishAsk(t, js, ask)
publishAsk(t, js, aDelivery(time.Now(), "/d"))
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy
t.Fatal(err)
}
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
var original *nats.Msg
for original == nil {
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
if err != nil || len(msgs) == 0 {
t.Fatalf("nothing to take: %v", err)
}
var r BuildRequest
_ = json.Unmarshal(msgs[0].Data, &r)
if r.ID == ask.ID {
original = msgs[0] // given up on, never settled: dead, and kept
continue
}
_ = msgs[0].Ack()
}
// Delivered again as DeliverAgain does: its headers, and its own.
again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}}
for k, v := range original.Header {
again.Header[k] = v
}
again.Header.Set(AgainHeader, "7")
if _, err := js.Context().PublishMsg(again); err != nil {
t.Fatal(err)
}
pass, err := PutInOrder(ctx, js, TheBuildMachine)
if err != nil || len(pass.Removed) != 0 {
t.Fatalf("removed %v: %v", pass.Removed, err)
}
n := 0
for _, a := range queued(t, js) {
if a.ID == ask.ID {
n++
}
}
if n != 2 {
t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n)
}
}
// Two passes at once: nothing is built twice, and nothing is lost.
func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
took, counts := takeAll(t, ctx, js)
now := time.Now()
const asks = 24
var passes sync.WaitGroup
for p := 0; p < 2; p++ {
passes.Add(1)
go func() {
defer passes.Done()
for i := 0; i < 15; i++ {
_, _ = PutInOrder(ctx, js, TheBuildMachine)
}
}()
}
for i := 0; i < asks; i++ {
repository := fmt.Sprintf("/%02d", i)
if i%3 == 2 {
publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository))
} else {
publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository))
}
}
passes.Wait()
for got := 0; got < asks; got++ {
select {
case <-took:
case <-time.After(15 * time.Second):
t.Fatalf("taken %d of %d: %v", got, asks, counts())
}
}
time.Sleep(time.Second)
c := counts()
for i := 0; i < asks; i++ {
if n := c[fmt.Sprintf("/%02d", i)]; n != 1 {
t.Errorf("/%02d was built %d times", i, n)
}
}
}
// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it.
func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
now := time.Now()
publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery"))
publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check"))
if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
t.Fatal(err)
}
took, _ := takeAll(t, ctx, js)
var order []string
for len(order) < 2 {
select {
case r := <-took:
order = append(order, r)
case <-time.After(10 * time.Second):
t.Fatalf("taken so far: %v", order)
}
}
if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) {
t.Fatalf("taken as %v, want %v", order, want)
}
}
// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when
// there was nothing left to delete.
func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
now := time.Now()
publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
read := queued(t, js)[0]
publishAsk(t, js, aDelivery(now, "/d"))
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
t.Fatalf("moved %v: %v", pass.Moved, err)
}
removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
if err != nil || removed == 0 || removed == read.Seq {
t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err)
}
for _, a := range queued(t, js) {
if a.ID == read.ID {
t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq)
}
}
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 {
t.Fatalf("an ask gone already is said removed at %d: %v", removed, err)
}
}
// An ask nothing can be read from keeps its place in time when it moves.
func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
t.Fatal(err)
}
first := queued(t, js)[0].Stored
time.Sleep(50 * time.Millisecond)
publishAsk(t, js, aDelivery(time.Now(), "/d"))
if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
t.Fatalf("moved %v: %v", pass.Moved, err)
}
asks := queued(t, js)
if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {
t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks)
}
}
internal/link/try_test.go
+2 -20
View File
@@ -2,7 +2,6 @@ package link
import (
"encoding/json"
"reflect"
"strings"
"testing"
"time"
@@ -30,25 +29,8 @@ func TestATrysStartNamesNoCommitTheForgeCouldMatch(t *testing.T) {
}
}
// A try stays behind every delivery and every pull request's check, however long it has waited: the ten
// minutes that put a check ahead of later deliveries never promote a try (novox/hq issue 686).
func TestATryIsNeverHandedOutBeforeADeliveryOrAPullRequestsCheck(t *testing.T) {
now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
try := func(seq uint64, id string, asked time.Time) QueuedAsk {
a := anAsk(seq, id, asked, true)
a.Request.Check.Try = true
return a
}
waiting := []QueuedAsk{
try(1, "try-old", now.Add(-40*time.Minute)),
try(2, "try-young", now.Add(-time.Minute)),
anAsk(3, "check-young", now.Add(-2*time.Minute), true),
anAsk(4, "delivery", now.Add(-time.Minute), false),
}
want := []string{"delivery", "check-young", "try-old", "try-young"}
if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
t.Fatalf("handed out as %v, want %v", got, want)
}
// An id that is a pattern never passes for a build's id.
func TestAnIdThatIsAPatternIsNoBuildId(t *testing.T) {
if !SafeBuildID("build-20261014T000000-1a2b") || SafeBuildID("x.>") || SafeBuildID("*") || SafeBuildID("") {
t.Fatal("an id that is a pattern passes for a build's id")
}
testdata/store-tests
Vendored
+4
View File
@@ -133,3 +133,7 @@
./cmd/mesh-controller TestAnOwnPathBatchNeverWalksBeside
./cmd/mesh-controller TestMergesHeardWhileAnotherHolderHasThePlansAreRecordedAndCarried
./cmd/mesh-controller TestABatchWindowClosesByTheExpiryOfItsKey
./cmd/mesh-controller TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid
./cmd/mesh-controller TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt
./cmd/mesh-controller TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent
./cmd/mesh-controller TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck