diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 62cb557d9..a0f2bb829 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "fmt" - mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "maps" "os" "path/filepath" @@ -20,6 +19,8 @@ import ( "testing" "time" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -37,6 +38,7 @@ import ( runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatdef" "git.novox.be/novox/mesh-sdk/go/views/viewstest" "mesh-delivery-builder/internal/build" @@ -299,6 +301,12 @@ func startBedWith(t *testing.T, closed bool) *bed { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, versions.BoundsState), History: 1, LimitMarkerTTL: time.Second}); err != nil { t.Fatal(err) } + // The seat mesh-builds' work queue and its one worker, as the broker derives them from the seat's definition (design 47 + // H2 §777), and the state taken the builder takes each ask once in (§775, §820). + b.seatQueue(t) + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.TakenState), History: 1, LimitMarkerTTL: time.Second}); err != nil { + t.Fatal(err) + } b.out = &saidLog{} // The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects. logStream := buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog) @@ -1252,3 +1260,29 @@ func (r *registry) numberedOf(name string) []string { } return out } + +// seatQueue makes the seat mesh-builds' work-queue stream and its worker from the seat's definition in the contracts +// module, as the broker derives them (buscompose.SeatQueue; design 47 H2 §777). +func (b *bed) seatQueue(t *testing.T) { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-delivery-contracts", "seats", meshbuilds.Seat, "seat.json")) + if err != nil { + t.Fatal(err) + } + def, err := seatdef.Parse(raw) + if err != nil { + t.Fatal(err) + } + stream, worker, ok, err := buscompose.SeatQueue(meshbuilds.Seat, def.Accepts) + if err != nil || !ok { + t.Fatalf("the seat's work queue: %v %v", ok, err) + } + if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: stream.Name, Subjects: stream.Subjects, Retention: jetstream.WorkQueuePolicy, + MaxAge: time.Duration(stream.MaxAge) * time.Second, MaxMsgs: int64(stream.MaxMsgs), Duplicates: time.Duration(stream.DuplicatesSeconds) * time.Second}); err != nil { + t.Fatal(err) + } + if _, err := b.js.CreateOrUpdateConsumer(b.ctx, stream.Name, jetstream.ConsumerConfig{Durable: worker.Name, FilterSubjects: worker.Filters, AckPolicy: jetstream.AckExplicitPolicy, + AckWait: time.Duration(worker.AckWaitSeconds) * time.Second, MaxDeliver: worker.MaxDeliver, MaxAckPending: worker.MaxAckPending}); err != nil { + t.Fatal(err) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 2f50c05af..af5ae06d4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -15,6 +15,7 @@ import ( "os/signal" "strconv" "strings" + "sync/atomic" "syscall" "time" @@ -37,6 +38,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/operatorask" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" @@ -259,7 +261,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro queueKV := process.AwaitBucket(ctx, js, conn, own.queue, "no version is published again on an event or by builder_publish_again", logger.Printf) actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf) republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}}, - Entry: entryVia(calls.Call), Send: sendVia(calls.Call), + Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk), Unreleased: reserved.unreleasedWords, Logf: logger.Printf, Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a) Started: time.Now()} // a build ended from now on is of the live class, sent first (H-A-290 (1)) @@ -486,7 +488,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("answering the verb reserved: %w", err) } defer stopReserved() - if err := serveSteps(ctx, conn, js, cred, b, logger.Printf); err != nil { + // The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken. + takenBound, err := askTakenBound() + if err != nil { + return err + } + var ackWait atomic.Int64 // the worker's ack wait, read when it is bound + ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait)) + steps := &stepQueue{node: cred.Node, builder: b, taken: calls.State(process.TakenState), takenBound: takenBound, + inProgress: inProgressOf(seatlist.Definition(seatGate.Reader, meshbuilds.Seat), func() time.Duration { return time.Duration(ackWait.Load()) }), + say: func(ctx context.Context, subject, id string, body any) error { + return emitSeatEvent(ctx, js, cred, subject, id, body) + }, logf: logger.Printf, now: time.Now, running: map[string]*runningAsk{}} + if err := serveSteps(ctx, conn, js, steps, &ackWait); err != nil { return err } seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx) }, logger.Printf) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go index 0ea48b011..09aebf982 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go @@ -2,15 +2,20 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "sync/atomic" + "time" "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/process" + "mesh-delivery-builder/internal/versions" ) @@ -46,12 +51,28 @@ func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (me } } +// keptCaller is a kept call carrying an ask id (process.Conn.CallKeptAsk). +type keptCaller func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) + +// publishAskID is the ask id of p's publish (design 47 H2 §775): the same seat, name, version and digest make the same id, +// so a publish made again is one the holder takes once (ADR 0668 §2); an id holds no dot, as every ask id. +func publishAskID(p versions.Publish) string { + sum := sha256.Sum256([]byte(p.Seat + "\n" + p.Name + "\n" + p.Version + "\n" + p.Digest)) + return "publish" + hex.EncodeToString(sum[:8]) +} + // sendVia sends a publish to the registry seat with the digest and no parts (A-73): the holder takes the bytes from the // artifact store, addressed by digest. The four seats' publish take one wire form; an answer that lacks parts, or a fault of -// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes). -func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) error { +// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes). The publish is a kept +// call carrying its ask id (design 47 H2 §775), made again through transport faults at the kept-call pace within ctx's bound. +func sendVia(call keptCaller) func(ctx context.Context, p versions.Publish) error { return func(ctx context.Context, p versions.Publish) error { - raw, err := callWithin(ctx, call, "seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p)) + ask := process.KeptAsk{ID: publishAskID(p)} + if d, ok := ctx.Deadline(); ok { + ask.Bound = time.Until(d) + } + raw, err := callWithin(ctx, func(key string, body any) (json.RawMessage, error) { return call(ask, key, body) }, + "seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p)) if err != nil { return fmt.Errorf("%s's publish of %s %s: %w", p.Seat, p.Name, p.Version, err) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go index 51770fed2..7a22acdb7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go @@ -13,6 +13,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry" + "git.novox.be/novox/mesh-sdk/go/process" + "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" ) @@ -48,12 +50,14 @@ func TestRepublishSend(t *testing.T) { "a fault": {"", errors.New("no responders"), "no responders"}, "no sense": {`[`, nil, "no publish answer"}, } { - send := sendVia(func(key string, body any) (json.RawMessage, error) { - asked = key + var id string + send := sendVia(func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) { + asked, id = key, ask.ID return json.RawMessage(c.answer), c.err }) err := send(context.Background(), p) - if asked != "seat:oci-artifact-registry.publish" || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) { + // design 47 H2 §775: the publish carries its ask id, the same for the same publish + if asked != "seat:oci-artifact-registry.publish" || id != publishAskID(p) || strings.Contains(id, ".") || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) { t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want) } } @@ -68,8 +72,10 @@ func TestRepublishSend(t *testing.T) { return json.RawMessage(`{}`), nil } for name, call := range map[string]func(context.Context) error{ - "the publish": func(ctx context.Context) error { return sendVia(slow)(ctx, p) }, - "the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err }, + "the publish": func(ctx context.Context) error { + return sendVia(func(_ process.KeptAsk, k string, b any) (json.RawMessage, error) { return slow(k, b) })(ctx, p) + }, + "the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err }, } { ctx, cancel := context.WithCancel(context.Background()) go func() { <-called; cancel() }() diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step.go index adc0a2bd3..72d74cc19 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step.go @@ -2,19 +2,11 @@ package main import ( "context" - "encoding/json" "fmt" "sync" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-sdk/go/process" - - "mesh-delivery-builder/internal/build" ) // **The runner's build step ask is the builder's one entry point for a candidate** (design 47 G A-97, A-84, A-99): the seat @@ -72,30 +64,3 @@ func (w *entryWaits) entry(ctx context.Context, fingerprint string) (meshcatalog } return e, w.manifest.keep(e) // the manifest the build is planned from } - -// serveSteps hears the step asks on the seat's accept subject and answers each one's end, in its own goroutine. -func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, cred process.Account, b *build.Builder, logf func(string, ...any)) error { - sub, err := conn.Subscribe(meshbuilds.AcceptSubjectStep, func(m *nats.Msg) { - id := m.Header.Get("x-event-id") - if id == "" { - id = m.Header.Get(nats.MsgIdHdr) - } - var run meshbuilds.StepRun - if err := json.Unmarshal(m.Data, &run); err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild { - logf("a step ask on %s that is no build step with an id is left: %v", m.Subject, err) - return - } - go func() { - end := b.BuildStep(ctx, run) - end.ID = id - if err := emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectStepEnded(cred.Node), id, end); err != nil { - logf("saying the end of the build step %s: %v", id, err) - } - }() - }) - if err != nil { - return err - } - go func() { <-ctx.Done(); _ = sub.Drain() }() - return nil -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go index 8251652b0..982b2cbc7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go @@ -4,14 +4,15 @@ package main import ( "encoding/json" + "runtime" "strings" "testing" "time" "github.com/nats-io/nats.go" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-sdk/go/buscompose" ) // The runner's build step ask on the seat mesh-builds' accept subject, naming an entry by its fingerprint, is built through the @@ -30,16 +31,33 @@ func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) { t.Fatal(err) } defer s.Unsubscribe() - b.start(t, "n1", nil) - b.await(t, "netcheck", isState(builds.BuildDone)) + taken := make(chan *nats.Msg, 4) + st, err := sub.ChanSubscribe(meshbuilds.SubjectAskTakenAll, taken) + if err != nil { + t.Fatal(err) + } + defer st.Unsubscribe() + // Design 47 H2 §777: an ask put on the seat's work queue while the builder is stopped waits there, and is taken (ask-taken) + // and ended once the builder starts. run := meshbuilds.StepRun{Pipeline: "main.1.netcheck", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 1, Entries: []meshbuilds.BuildEntry{{Module: "netcheck", Fingerprint: fp}}} data, _ := json.Marshal(run) msg := nats.NewMsg(meshbuilds.AcceptSubjectStep) msg.Data = data msg.Header.Set("x-event-id", "ask-1") - if err := sub.PublishMsg(msg); err != nil { - t.Fatal(err) + msg.Header.Set(nats.MsgIdHdr, "ask-1") + if _, err := b.js.PublishMsg(b.ctx, msg); err != nil { + t.Fatalf("the ask is not put on the seat's work queue: %v", err) + } + b.start(t, "n1", nil) + select { + case m := <-taken: + var at meshbuilds.AskTakenEvent + if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "ask-1" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshbuilds.SubjectAskTaken("ask-1") { + t.Fatalf("ask-taken on %s = %+v, %v; want ask-1 taken on n1", m.Subject, at, err) + } + case <-time.After(20 * time.Second): + t.Fatal("no ask-taken within 20 s of the start") } select { case m := <-ended: @@ -75,4 +93,24 @@ func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) { case <-time.After(20 * time.Second): t.Fatal("no step-ended for ask-2 within 20 s") } + // Each ask is acknowledged once its end was said: the seat's worker holds none pending. + stream, worker := buscompose.SeatQueueStreamName(meshbuilds.Seat), buscompose.SeatQueueWorkerName(meshbuilds.Seat) + deadline := time.Now().Add(10 * time.Second) + for { + c, err := b.js.Consumer(b.ctx, stream, worker) + if err != nil { + t.Fatal(err) + } + info, err := c.Info(b.ctx) + if err != nil { + t.Fatal(err) + } + if info.NumPending == 0 && info.NumAckPending == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("the seat's worker holds %d waiting and %d taken after both asks ended, want none", info.NumPending, info.NumAckPending) + } + runtime.Gosched() + } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepqueue.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepqueue.go new file mode 100644 index 000000000..7186ba673 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepqueue.go @@ -0,0 +1,240 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// **The seat mesh-builds takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775; ADR 0525 §1, §2, +// §6; ADR 0668 §2): the broker derives the seat's stream and its one shared worker from the seat's definition; the builder +// pulls every ask waiting at once, takes each once by its id in its state taken (TakenKey ., for +// ask-taken-bound; H-A-381 (2)), says ask-taken at the pull, sends in-progress while it builds, and acknowledges an ask +// when it said the ask's end. An ask put while the builder is down waits on the stream. + +// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the +// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default. +const ( + askTakenBoundSetting = "ask-taken-bound" + defaultAskTakenBound = 10 * time.Minute +) + +// askTaker is the state taken (process.State.TakeOnce). +type askTaker interface { + TakeOnce(seat, ask string, bound time.Duration) (bool, error) +} + +// stepBuilder is build.Builder's BuildStep. +type stepBuilder interface { + BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbuilds.StepEnded +} + +// queueMsg is what the queue reads of a message pulled from the seat's work queue (jetstream.Msg). +type queueMsg interface { + Data() []byte + Headers() nats.Header + Ack() error + Nak() error + Term() error + InProgress() error +} + +// stepQueue is the builder's pull of the seat mesh-builds' work queue. +type stepQueue struct { + node string + builder stepBuilder + taken askTaker + takenBound time.Duration + // inProgress is how often a running ask is said in progress: the seat definition's in-progress. + inProgress func() time.Duration + // say publishes a seat event of the builder (emitSeatEvent). + say func(ctx context.Context, subject, id string, body any) error + logf func(string, ...any) + now func() time.Time + + mu sync.Mutex + running map[string]*runningAsk +} + +// runningAsk is an ask the builder took and builds now. +type runningAsk struct { + run meshbuilds.StepRun + taken time.Time + cancel context.CancelCauseFunc +} + +// askIDOf is the ask's id a put carries: its event id, else its message id. +func askIDOf(h nats.Header) string { + if id := h.Get("x-event-id"); id != "" { + return id + } + return h.Get(nats.MsgIdHdr) +} + +// take handles one ask pulled from the queue: an ask that is no build step with an id is terminated, said; one this +// builder runs already (handed again while it runs) is left to the run that holds it; any other is taken (ask-taken said +// the first time its id is taken) and built on its own goroutine, in progress while it runs, acknowledged once its end is +// said. A repeat of an id taken before that runs here no longer (the builder started again) is answered as its builds +// stand now, read afresh by BuildStep (H-A-381 (1)): an entry built is answered built, never built twice. +func (q *stepQueue) take(ctx context.Context, m queueMsg) { + id := askIDOf(m.Headers()) + var run meshbuilds.StepRun + err := json.Unmarshal(m.Data(), &run) + if err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild { + q.logf("a step ask on the seat %s's work queue that is no build step with an id is ended: %v", meshbuilds.Seat, err) + _ = m.Term() + return + } + q.mu.Lock() + if _, held := q.running[id]; held { + q.mu.Unlock() + q.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline) + return + } + first, err := q.taken.TakeOnce(meshbuilds.Seat, id, q.takenBound) + if err != nil { + q.mu.Unlock() + q.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err) + _ = m.Nak() + return + } + work, cancel := context.WithCancelCause(ctx) + ra := &runningAsk{run: run, taken: q.now(), cancel: cancel} + q.running[id] = ra + q.mu.Unlock() + if first { + if err := q.say(ctx, meshbuilds.SubjectAskTaken(id), id+"-taken", meshbuilds.AskTakenEvent{Ask: id, Node: q.node, At: ra.taken.UTC()}); err != nil { + q.logf("saying ask-taken of %s: %v", id, err) + } + } else { + q.logf("the ask %s (%s) was taken before: it is answered as its builds stand now", id, run.Pipeline) + } + go q.build(work, m, id, ra) +} + +// build runs one taken ask to its end and says it. +func (q *stepQueue) build(ctx context.Context, m queueMsg, id string, ra *runningAsk) { + defer func() { + ra.cancel(nil) + q.mu.Lock() + delete(q.running, id) + q.mu.Unlock() + }() + every := q.inProgress() + var tick <-chan time.Time + if every > 0 { + t := time.NewTicker(every) + defer t.Stop() + tick = t.C + } + done := make(chan meshbuilds.StepEnded, 1) + go func() { done <- q.builder.BuildStep(ctx, ra.run) }() + var end meshbuilds.StepEnded + for waiting := true; waiting; { + select { + case end = <-done: + waiting = false + case <-tick: + _ = m.InProgress() + } + } + end.ID = id + if err := q.say(context.WithoutCancel(ctx), meshbuilds.SubjectStepEnded(q.node), id, end); err != nil { + q.logf("saying the end of the build step %s: %v; the ask is handed again", id, err) + _ = m.Nak() + return + } + _ = m.Ack() +} + +// running answers the asks this builder runs now, by id. +func (q *stepQueue) runningAsks() map[string]runningAsk { + q.mu.Lock() + defer q.mu.Unlock() + out := make(map[string]runningAsk, len(q.running)) + for id, ra := range q.running { + out[id] = *ra + } + return out +} + +// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the +// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777). +func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration { + return func() time.Duration { + if d, ok := def(); ok { + if _, worker, ok, err := buscompose.SeatQueue(meshbuilds.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 { + return time.Duration(worker.InProgressSeconds) * time.Second + } + } + return ackWait() / 3 + } +} + +// serveSteps pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's +// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created. +func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, q *stepQueue, ackWait *atomic.Int64) error { + stream, worker := buscompose.SeatQueueStreamName(meshbuilds.Seat), buscompose.SeatQueueWorkerName(meshbuilds.Seat) + var once sync.Once + bind := func() bool { + c, err := js.Consumer(ctx, stream, worker) + if err != nil { + return false + } + bound := false + once.Do(func() { + bound = true + info := c.CachedInfo() + if info != nil && info.Config.AckWait > 0 { + ackWait.Store(int64(info.Config.AckWait)) + } + pending := 1000 + if info != nil && info.Config.MaxAckPending > 0 { + pending = info.Config.MaxAckPending + } + cc, err := c.Consume(func(m jetstream.Msg) { q.take(ctx, m) }, jetstream.PullMaxMessages(pending)) + if err != nil { + q.logf("the seat %s's work queue %s is not pulled: %v", meshbuilds.Seat, worker, err) + return + } + q.logf("pulling the seat %s's asks from %s", meshbuilds.Seat, worker) + go func() { <-ctx.Done(); cc.Stop() }() + }) + return bound + } + if bind() { + return nil + } + q.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshbuilds.Seat, worker, stream) + sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() }) + if err != nil { + return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshbuilds.Seat, err) + } + go func() { <-ctx.Done(); _ = sub.Unsubscribe() }() + if bind() { // made between the first look and the subscription + _ = sub.Unsubscribe() + } + return nil +} + +// askTakenBound reads the setting ask-taken-bound (§781). +func askTakenBound() (time.Duration, error) { + d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound) + if err != nil { + return 0, errors.New("the builder's " + err.Error()) + } + return d, nil +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 2574b54d4..3bd70a2a3 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -75,6 +75,7 @@ type manifest struct { Maximum float64 `json:"maximum"` Trusted bool `json:"trusted"` Default any `json:"default"` + Runtime bool `json:"runtime"` } `json:"config"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` @@ -114,7 +115,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -186,7 +187,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams) } - if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 7 || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) || + if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 10 || + // design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process + m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" || + !m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) || m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(8) { t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 8 tries (design 47 G §588, §633, A-457a)", m.Config) } @@ -306,7 +310,9 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR } for _, f := range forbidden { // The one subscription is the seat mesh-builds' accepted kind `step`, the runner's build ask (design 47 G A-97, A-107). - if f == "Subscribe(" && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/step.go" { + // Design 47 H2 §777: the seat's work queue is pulled, its worker bound when the bus says it was created (the + // one subscription), and a taken ask is said in progress while it is built (ADR 0525 §6), the queue's own cadence. + if (f == "Subscribe(" || f == "time.NewTicker") && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/stepqueue.go" { continue } if strings.Contains(string(src), f) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index f58f40c88..30ac44c5c 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -123,6 +123,7 @@ { "name": "instances", "history": 1 }, { "name": "old-form", "history": 1 }, { "name": "bounds", "history": 1, "key-ttl": true }, + { "name": "taken", "history": 1, "key-ttl": true, "max-mib": 8 }, { "name": "queues", "history": 1 }, { "name": "toolchains-heard", "history": 1 }, { "name": "waits", "history": 1, "max-mib": 4 }, @@ -164,6 +165,27 @@ "trusted": false, "default": 10 }, + "ask-taken-bound": { + "type": "string", + "description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).", + "trusted": false, + "default": "10m", + "runtime": true + }, + "kept-call-pace-first": { + "type": "string", + "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).", + "trusted": false, + "default": "1s", + "runtime": true + }, + "kept-call-pace-max": { + "type": "string", + "description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).", + "trusted": false, + "default": "30s", + "runtime": true + }, "publish-pause": { "type": "number", "description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).",