B1: the builder pulls the seat mesh-builds' work queue and takes each ask once
An ask put while the builder is down waited for nobody: the plain subscription heard only what came while it ran. The builder now pulls every waiting ask from the seat's one worker, takes it once by its id in its state taken (for ask-taken-bound), says ask-taken at the pull, sends in-progress while it builds and acknowledges once the step's end is said. Its publish to a registry seat is a kept call carrying the publish's ask id. The manifest declares taken and the kept-call pace and ask-taken-bound as runtime settings. Builds: design 47 H2 §775, §777, §820; H-A-381
This commit is contained in:
@@ -9,7 +9,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -20,6 +19,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -37,6 +38,7 @@ import (
|
||||
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
@@ -299,6 +301,12 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, versions.BoundsState), History: 1, LimitMarkerTTL: time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The seat mesh-builds' work queue and its one worker, as the broker derives them from the seat's definition (design 47
|
||||
// H2 §777), and the state taken the builder takes each ask once in (§775, §820).
|
||||
b.seatQueue(t)
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.TakenState), History: 1, LimitMarkerTTL: time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.out = &saidLog{}
|
||||
// The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects.
|
||||
logStream := buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog)
|
||||
@@ -1252,3 +1260,29 @@ func (r *registry) numberedOf(name string) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seatQueue makes the seat mesh-builds' work-queue stream and its worker from the seat's definition in the contracts
|
||||
// module, as the broker derives them (buscompose.SeatQueue; design 47 H2 §777).
|
||||
func (b *bed) seatQueue(t *testing.T) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-delivery-contracts", "seats", meshbuilds.Seat, "seat.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
def, err := seatdef.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream, worker, ok, err := buscompose.SeatQueue(meshbuilds.Seat, def.Accepts)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("the seat's work queue: %v %v", ok, err)
|
||||
}
|
||||
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: stream.Name, Subjects: stream.Subjects, Retention: jetstream.WorkQueuePolicy,
|
||||
MaxAge: time.Duration(stream.MaxAge) * time.Second, MaxMsgs: int64(stream.MaxMsgs), Duplicates: time.Duration(stream.DuplicatesSeconds) * time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, stream.Name, jetstream.ConsumerConfig{Durable: worker.Name, FilterSubjects: worker.Filters, AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: time.Duration(worker.AckWaitSeconds) * time.Second, MaxDeliver: worker.MaxDeliver, MaxAckPending: worker.MaxAckPending}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -37,6 +38,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/operatorask"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
@@ -259,7 +261,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
queueKV := process.AwaitBucket(ctx, js, conn, own.queue, "no version is published again on an event or by builder_publish_again", logger.Printf)
|
||||
actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf)
|
||||
republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}},
|
||||
Entry: entryVia(calls.Call), Send: sendVia(calls.Call),
|
||||
Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk),
|
||||
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
|
||||
Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
|
||||
Started: time.Now()} // a build ended from now on is of the live class, sent first (H-A-290 (1))
|
||||
@@ -486,7 +488,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return fmt.Errorf("answering the verb reserved: %w", err)
|
||||
}
|
||||
defer stopReserved()
|
||||
if err := serveSteps(ctx, conn, js, cred, b, logger.Printf); err != nil {
|
||||
// The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken.
|
||||
takenBound, err := askTakenBound()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var ackWait atomic.Int64 // the worker's ack wait, read when it is bound
|
||||
ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait))
|
||||
steps := &stepQueue{node: cred.Node, builder: b, taken: calls.State(process.TakenState), takenBound: takenBound,
|
||||
inProgress: inProgressOf(seatlist.Definition(seatGate.Reader, meshbuilds.Seat), func() time.Duration { return time.Duration(ackWait.Load()) }),
|
||||
say: func(ctx context.Context, subject, id string, body any) error {
|
||||
return emitSeatEvent(ctx, js, cred, subject, id, body)
|
||||
}, logf: logger.Printf, now: time.Now, running: map[string]*runningAsk{}}
|
||||
if err := serveSteps(ctx, conn, js, steps, &ackWait); err != nil {
|
||||
return err
|
||||
}
|
||||
seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx) }, logger.Printf)
|
||||
|
||||
@@ -2,15 +2,20 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
@@ -46,12 +51,28 @@ func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (me
|
||||
}
|
||||
}
|
||||
|
||||
// keptCaller is a kept call carrying an ask id (process.Conn.CallKeptAsk).
|
||||
type keptCaller func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
|
||||
|
||||
// publishAskID is the ask id of p's publish (design 47 H2 §775): the same seat, name, version and digest make the same id,
|
||||
// so a publish made again is one the holder takes once (ADR 0668 §2); an id holds no dot, as every ask id.
|
||||
func publishAskID(p versions.Publish) string {
|
||||
sum := sha256.Sum256([]byte(p.Seat + "\n" + p.Name + "\n" + p.Version + "\n" + p.Digest))
|
||||
return "publish" + hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// sendVia sends a publish to the registry seat with the digest and no parts (A-73): the holder takes the bytes from the
|
||||
// artifact store, addressed by digest. The four seats' publish take one wire form; an answer that lacks parts, or a fault of
|
||||
// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes).
|
||||
func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) error {
|
||||
// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes). The publish is a kept
|
||||
// call carrying its ask id (design 47 H2 §775), made again through transport faults at the kept-call pace within ctx's bound.
|
||||
func sendVia(call keptCaller) func(ctx context.Context, p versions.Publish) error {
|
||||
return func(ctx context.Context, p versions.Publish) error {
|
||||
raw, err := callWithin(ctx, call, "seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p))
|
||||
ask := process.KeptAsk{ID: publishAskID(p)}
|
||||
if d, ok := ctx.Deadline(); ok {
|
||||
ask.Bound = time.Until(d)
|
||||
}
|
||||
raw, err := callWithin(ctx, func(key string, body any) (json.RawMessage, error) { return call(ask, key, body) },
|
||||
"seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's publish of %s %s: %w", p.Seat, p.Name, p.Version, err)
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
@@ -48,12 +50,14 @@ func TestRepublishSend(t *testing.T) {
|
||||
"a fault": {"", errors.New("no responders"), "no responders"},
|
||||
"no sense": {`[`, nil, "no publish answer"},
|
||||
} {
|
||||
send := sendVia(func(key string, body any) (json.RawMessage, error) {
|
||||
asked = key
|
||||
var id string
|
||||
send := sendVia(func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
asked, id = key, ask.ID
|
||||
return json.RawMessage(c.answer), c.err
|
||||
})
|
||||
err := send(context.Background(), p)
|
||||
if asked != "seat:oci-artifact-registry.publish" || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) {
|
||||
// design 47 H2 §775: the publish carries its ask id, the same for the same publish
|
||||
if asked != "seat:oci-artifact-registry.publish" || id != publishAskID(p) || strings.Contains(id, ".") || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) {
|
||||
t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want)
|
||||
}
|
||||
}
|
||||
@@ -68,8 +72,10 @@ func TestRepublishSend(t *testing.T) {
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
for name, call := range map[string]func(context.Context) error{
|
||||
"the publish": func(ctx context.Context) error { return sendVia(slow)(ctx, p) },
|
||||
"the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err },
|
||||
"the publish": func(ctx context.Context) error {
|
||||
return sendVia(func(_ process.KeptAsk, k string, b any) (json.RawMessage, error) { return slow(k, b) })(ctx, p)
|
||||
},
|
||||
"the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err },
|
||||
} {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() { <-called; cancel() }()
|
||||
|
||||
@@ -2,19 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// **The runner's build step ask is the builder's one entry point for a candidate** (design 47 G A-97, A-84, A-99): the seat
|
||||
@@ -72,30 +64,3 @@ func (w *entryWaits) entry(ctx context.Context, fingerprint string) (meshcatalog
|
||||
}
|
||||
return e, w.manifest.keep(e) // the manifest the build is planned from
|
||||
}
|
||||
|
||||
// serveSteps hears the step asks on the seat's accept subject and answers each one's end, in its own goroutine.
|
||||
func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, cred process.Account, b *build.Builder, logf func(string, ...any)) error {
|
||||
sub, err := conn.Subscribe(meshbuilds.AcceptSubjectStep, func(m *nats.Msg) {
|
||||
id := m.Header.Get("x-event-id")
|
||||
if id == "" {
|
||||
id = m.Header.Get(nats.MsgIdHdr)
|
||||
}
|
||||
var run meshbuilds.StepRun
|
||||
if err := json.Unmarshal(m.Data, &run); err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild {
|
||||
logf("a step ask on %s that is no build step with an id is left: %v", m.Subject, err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
end := b.BuildStep(ctx, run)
|
||||
end.ID = id
|
||||
if err := emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectStepEnded(cred.Node), id, end); err != nil {
|
||||
logf("saying the end of the build step %s: %v", id, err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { <-ctx.Done(); _ = sub.Drain() }()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,14 +4,15 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// The runner's build step ask on the seat mesh-builds' accept subject, naming an entry by its fingerprint, is built through the
|
||||
@@ -30,16 +31,33 @@ func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Unsubscribe()
|
||||
b.start(t, "n1", nil)
|
||||
b.await(t, "netcheck", isState(builds.BuildDone))
|
||||
taken := make(chan *nats.Msg, 4)
|
||||
st, err := sub.ChanSubscribe(meshbuilds.SubjectAskTakenAll, taken)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Unsubscribe()
|
||||
// Design 47 H2 §777: an ask put on the seat's work queue while the builder is stopped waits there, and is taken (ask-taken)
|
||||
// and ended once the builder starts.
|
||||
run := meshbuilds.StepRun{Pipeline: "main.1.netcheck", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 1,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "netcheck", Fingerprint: fp}}}
|
||||
data, _ := json.Marshal(run)
|
||||
msg := nats.NewMsg(meshbuilds.AcceptSubjectStep)
|
||||
msg.Data = data
|
||||
msg.Header.Set("x-event-id", "ask-1")
|
||||
if err := sub.PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
msg.Header.Set(nats.MsgIdHdr, "ask-1")
|
||||
if _, err := b.js.PublishMsg(b.ctx, msg); err != nil {
|
||||
t.Fatalf("the ask is not put on the seat's work queue: %v", err)
|
||||
}
|
||||
b.start(t, "n1", nil)
|
||||
select {
|
||||
case m := <-taken:
|
||||
var at meshbuilds.AskTakenEvent
|
||||
if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "ask-1" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshbuilds.SubjectAskTaken("ask-1") {
|
||||
t.Fatalf("ask-taken on %s = %+v, %v; want ask-1 taken on n1", m.Subject, at, err)
|
||||
}
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("no ask-taken within 20 s of the start")
|
||||
}
|
||||
select {
|
||||
case m := <-ended:
|
||||
@@ -75,4 +93,24 @@ func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) {
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("no step-ended for ask-2 within 20 s")
|
||||
}
|
||||
// Each ask is acknowledged once its end was said: the seat's worker holds none pending.
|
||||
stream, worker := buscompose.SeatQueueStreamName(meshbuilds.Seat), buscompose.SeatQueueWorkerName(meshbuilds.Seat)
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
c, err := b.js.Consumer(b.ctx, stream, worker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := c.Info(b.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumPending == 0 && info.NumAckPending == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the seat's worker holds %d waiting and %d taken after both asks ended, want none", info.NumPending, info.NumAckPending)
|
||||
}
|
||||
runtime.Gosched()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// **The seat mesh-builds takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775; ADR 0525 §1, §2,
|
||||
// §6; ADR 0668 §2): the broker derives the seat's stream and its one shared worker from the seat's definition; the builder
|
||||
// pulls every ask waiting at once, takes each once by its id in its state taken (TakenKey <seat>.<ask>, for
|
||||
// ask-taken-bound; H-A-381 (2)), says ask-taken at the pull, sends in-progress while it builds, and acknowledges an ask
|
||||
// when it said the ask's end. An ask put while the builder is down waits on the stream.
|
||||
|
||||
// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the
|
||||
// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default.
|
||||
const (
|
||||
askTakenBoundSetting = "ask-taken-bound"
|
||||
defaultAskTakenBound = 10 * time.Minute
|
||||
)
|
||||
|
||||
// askTaker is the state taken (process.State.TakeOnce).
|
||||
type askTaker interface {
|
||||
TakeOnce(seat, ask string, bound time.Duration) (bool, error)
|
||||
}
|
||||
|
||||
// stepBuilder is build.Builder's BuildStep.
|
||||
type stepBuilder interface {
|
||||
BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbuilds.StepEnded
|
||||
}
|
||||
|
||||
// queueMsg is what the queue reads of a message pulled from the seat's work queue (jetstream.Msg).
|
||||
type queueMsg interface {
|
||||
Data() []byte
|
||||
Headers() nats.Header
|
||||
Ack() error
|
||||
Nak() error
|
||||
Term() error
|
||||
InProgress() error
|
||||
}
|
||||
|
||||
// stepQueue is the builder's pull of the seat mesh-builds' work queue.
|
||||
type stepQueue struct {
|
||||
node string
|
||||
builder stepBuilder
|
||||
taken askTaker
|
||||
takenBound time.Duration
|
||||
// inProgress is how often a running ask is said in progress: the seat definition's in-progress.
|
||||
inProgress func() time.Duration
|
||||
// say publishes a seat event of the builder (emitSeatEvent).
|
||||
say func(ctx context.Context, subject, id string, body any) error
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
running map[string]*runningAsk
|
||||
}
|
||||
|
||||
// runningAsk is an ask the builder took and builds now.
|
||||
type runningAsk struct {
|
||||
run meshbuilds.StepRun
|
||||
taken time.Time
|
||||
cancel context.CancelCauseFunc
|
||||
}
|
||||
|
||||
// askIDOf is the ask's id a put carries: its event id, else its message id.
|
||||
func askIDOf(h nats.Header) string {
|
||||
if id := h.Get("x-event-id"); id != "" {
|
||||
return id
|
||||
}
|
||||
return h.Get(nats.MsgIdHdr)
|
||||
}
|
||||
|
||||
// take handles one ask pulled from the queue: an ask that is no build step with an id is terminated, said; one this
|
||||
// builder runs already (handed again while it runs) is left to the run that holds it; any other is taken (ask-taken said
|
||||
// the first time its id is taken) and built on its own goroutine, in progress while it runs, acknowledged once its end is
|
||||
// said. A repeat of an id taken before that runs here no longer (the builder started again) is answered as its builds
|
||||
// stand now, read afresh by BuildStep (H-A-381 (1)): an entry built is answered built, never built twice.
|
||||
func (q *stepQueue) take(ctx context.Context, m queueMsg) {
|
||||
id := askIDOf(m.Headers())
|
||||
var run meshbuilds.StepRun
|
||||
err := json.Unmarshal(m.Data(), &run)
|
||||
if err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild {
|
||||
q.logf("a step ask on the seat %s's work queue that is no build step with an id is ended: %v", meshbuilds.Seat, err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
if _, held := q.running[id]; held {
|
||||
q.mu.Unlock()
|
||||
q.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline)
|
||||
return
|
||||
}
|
||||
first, err := q.taken.TakeOnce(meshbuilds.Seat, id, q.takenBound)
|
||||
if err != nil {
|
||||
q.mu.Unlock()
|
||||
q.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
work, cancel := context.WithCancelCause(ctx)
|
||||
ra := &runningAsk{run: run, taken: q.now(), cancel: cancel}
|
||||
q.running[id] = ra
|
||||
q.mu.Unlock()
|
||||
if first {
|
||||
if err := q.say(ctx, meshbuilds.SubjectAskTaken(id), id+"-taken", meshbuilds.AskTakenEvent{Ask: id, Node: q.node, At: ra.taken.UTC()}); err != nil {
|
||||
q.logf("saying ask-taken of %s: %v", id, err)
|
||||
}
|
||||
} else {
|
||||
q.logf("the ask %s (%s) was taken before: it is answered as its builds stand now", id, run.Pipeline)
|
||||
}
|
||||
go q.build(work, m, id, ra)
|
||||
}
|
||||
|
||||
// build runs one taken ask to its end and says it.
|
||||
func (q *stepQueue) build(ctx context.Context, m queueMsg, id string, ra *runningAsk) {
|
||||
defer func() {
|
||||
ra.cancel(nil)
|
||||
q.mu.Lock()
|
||||
delete(q.running, id)
|
||||
q.mu.Unlock()
|
||||
}()
|
||||
every := q.inProgress()
|
||||
var tick <-chan time.Time
|
||||
if every > 0 {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
tick = t.C
|
||||
}
|
||||
done := make(chan meshbuilds.StepEnded, 1)
|
||||
go func() { done <- q.builder.BuildStep(ctx, ra.run) }()
|
||||
var end meshbuilds.StepEnded
|
||||
for waiting := true; waiting; {
|
||||
select {
|
||||
case end = <-done:
|
||||
waiting = false
|
||||
case <-tick:
|
||||
_ = m.InProgress()
|
||||
}
|
||||
}
|
||||
end.ID = id
|
||||
if err := q.say(context.WithoutCancel(ctx), meshbuilds.SubjectStepEnded(q.node), id, end); err != nil {
|
||||
q.logf("saying the end of the build step %s: %v; the ask is handed again", id, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// running answers the asks this builder runs now, by id.
|
||||
func (q *stepQueue) runningAsks() map[string]runningAsk {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
out := make(map[string]runningAsk, len(q.running))
|
||||
for id, ra := range q.running {
|
||||
out[id] = *ra
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the
|
||||
// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777).
|
||||
func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration {
|
||||
return func() time.Duration {
|
||||
if d, ok := def(); ok {
|
||||
if _, worker, ok, err := buscompose.SeatQueue(meshbuilds.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 {
|
||||
return time.Duration(worker.InProgressSeconds) * time.Second
|
||||
}
|
||||
}
|
||||
return ackWait() / 3
|
||||
}
|
||||
}
|
||||
|
||||
// serveSteps pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's
|
||||
// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created.
|
||||
func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, q *stepQueue, ackWait *atomic.Int64) error {
|
||||
stream, worker := buscompose.SeatQueueStreamName(meshbuilds.Seat), buscompose.SeatQueueWorkerName(meshbuilds.Seat)
|
||||
var once sync.Once
|
||||
bind := func() bool {
|
||||
c, err := js.Consumer(ctx, stream, worker)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
bound := false
|
||||
once.Do(func() {
|
||||
bound = true
|
||||
info := c.CachedInfo()
|
||||
if info != nil && info.Config.AckWait > 0 {
|
||||
ackWait.Store(int64(info.Config.AckWait))
|
||||
}
|
||||
pending := 1000
|
||||
if info != nil && info.Config.MaxAckPending > 0 {
|
||||
pending = info.Config.MaxAckPending
|
||||
}
|
||||
cc, err := c.Consume(func(m jetstream.Msg) { q.take(ctx, m) }, jetstream.PullMaxMessages(pending))
|
||||
if err != nil {
|
||||
q.logf("the seat %s's work queue %s is not pulled: %v", meshbuilds.Seat, worker, err)
|
||||
return
|
||||
}
|
||||
q.logf("pulling the seat %s's asks from %s", meshbuilds.Seat, worker)
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
})
|
||||
return bound
|
||||
}
|
||||
if bind() {
|
||||
return nil
|
||||
}
|
||||
q.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshbuilds.Seat, worker, stream)
|
||||
sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() })
|
||||
if err != nil {
|
||||
return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshbuilds.Seat, err)
|
||||
}
|
||||
go func() { <-ctx.Done(); _ = sub.Unsubscribe() }()
|
||||
if bind() { // made between the first look and the subscription
|
||||
_ = sub.Unsubscribe()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// askTakenBound reads the setting ask-taken-bound (§781).
|
||||
func askTakenBound() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound)
|
||||
if err != nil {
|
||||
return 0, errors.New("the builder's " + err.Error())
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
@@ -75,6 +75,7 @@ type manifest struct {
|
||||
Maximum float64 `json:"maximum"`
|
||||
Trusted bool `json:"trusted"`
|
||||
Default any `json:"default"`
|
||||
Runtime bool `json:"runtime"`
|
||||
} `json:"config"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
@@ -114,7 +115,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
@@ -186,7 +187,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
|
||||
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
|
||||
}
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 7 || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 10 ||
|
||||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
|
||||
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
|
||||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(8) {
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 8 tries (design 47 G §588, §633, A-457a)", m.Config)
|
||||
}
|
||||
@@ -306,7 +310,9 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
// The one subscription is the seat mesh-builds' accepted kind `step`, the runner's build ask (design 47 G A-97, A-107).
|
||||
if f == "Subscribe(" && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/step.go" {
|
||||
// Design 47 H2 §777: the seat's work queue is pulled, its worker bound when the bus says it was created (the
|
||||
// one subscription), and a taken ask is said in progress while it is built (ADR 0525 §6), the queue's own cadence.
|
||||
if (f == "Subscribe(" || f == "time.NewTicker") && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/stepqueue.go" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(string(src), f) {
|
||||
|
||||
@@ -123,6 +123,7 @@
|
||||
{ "name": "instances", "history": 1 },
|
||||
{ "name": "old-form", "history": 1 },
|
||||
{ "name": "bounds", "history": 1, "key-ttl": true },
|
||||
{ "name": "taken", "history": 1, "key-ttl": true, "max-mib": 8 },
|
||||
{ "name": "queues", "history": 1 },
|
||||
{ "name": "toolchains-heard", "history": 1 },
|
||||
{ "name": "waits", "history": 1, "max-mib": 4 },
|
||||
@@ -164,6 +165,27 @@
|
||||
"trusted": false,
|
||||
"default": 10
|
||||
},
|
||||
"ask-taken-bound": {
|
||||
"type": "string",
|
||||
"description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).",
|
||||
"trusted": false,
|
||||
"default": "10m",
|
||||
"runtime": true
|
||||
},
|
||||
"kept-call-pace-first": {
|
||||
"type": "string",
|
||||
"description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).",
|
||||
"trusted": false,
|
||||
"default": "1s",
|
||||
"runtime": true
|
||||
},
|
||||
"kept-call-pace-max": {
|
||||
"type": "string",
|
||||
"description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).",
|
||||
"trusted": false,
|
||||
"default": "30s",
|
||||
"runtime": true
|
||||
},
|
||||
"publish-pause": {
|
||||
"type": "number",
|
||||
"description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).",
|
||||
|
||||
Reference in New Issue
Block a user