From b6a7e0bad3b29dd385d85b3ccffc3a4e3a039ebe Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:35:32 +0100 Subject: [PATCH] B4 (H-A-454): publishes is the builder's own work-queue stream, read by the verb publish-queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The work queue publishes is declared in the manifest's old streams section (retention work-queue, duplicates 24 h, max-deliveries 10, 64 MiB, 7 days), the module's own stream consumed on the module's account, so the bus binding asked of the broker seat goes; the consumers per holder and class are still made at run time. The seat mesh-builds gains the read verb publish-queue, served by the builder from its own stream (counts per holder and class, oldest and newest, a list capped at 50, capped past 4,000 items read), and builder_publish_queue asks it in place of the runtime's stream and work-queue reads. Two whole tests of a gone artifact are folded into one, so the builder's integration count stays at 18 with the publishes whole test. Builds: design 47 H2 §793, §819, §820; H-A-440 (1), H-A-446 (2), H-A-454 and its addendum --- .../cmd/mesh-delivery-builder-tools/main.go | 2 +- .../mesh-delivery-builder-tools/republish.go | 70 +++------ .../republish_test.go | 46 +++--- .../gone_integration_test.go | 11 +- .../cmd/mesh-delivery-builder/main.go | 19 +-- .../publishes_integration_test.go | 18 ++- .../cmd/mesh-delivery-builder/publishitem.go | 87 ++++++++++- modules/mesh-delivery-builder/go.mod | 1 - .../internal/versions/republish.go | 41 +++-- .../internal/versions/republish_test.go | 29 +++- .../mesh-delivery-builder/manifest_test.go | 25 ++- modules/mesh-delivery-builder/module.json | 10 +- .../go/seats/meshbuilds/contract.go | 92 ++++++++++- .../seats/mesh-builds/seat.json | 144 ++++++++++++++++++ .../ts/seats/meshbuilds/index.d.ts | 59 +++++++ .../ts/seats/meshbuilds/index.js | 1 + 16 files changed, 520 insertions(+), 135 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index cf230b605..225c2789f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -435,7 +435,7 @@ func main() { tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) as := stdio.State(versions.ActedState) tools = append(tools, publishActTools(buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) - tools = append(tools, republishTools(workQueueReader(stdio.BusState, versions.PublishesLocal), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + tools = append(tools, republishTools(publishQueueVia(stdio.Ask), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) ws := stdio.State(build.WaitsState) wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := ws.Get(key) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go index 91c73783d..86a0841c0 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go @@ -4,13 +4,12 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "strings" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/buscalls" "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" @@ -55,54 +54,23 @@ func onePage(page func(after string, maxBytes int, budget time.Duration) (stdio. return out, nil } -// workQueuePageItems and workQueueItemsRead bound the read of the work queue publishes: pages of 500 items, at most 4,000 read -// (H-A-293: past them the answer is capped, naming the items read and the queue's size). -const ( - workQueuePageItems = 500 - workQueueItemsRead = 4000 -) +// publishQueueVerbKey is the seat verb builder_publish_queue asks (the manifest's invokes): the builder reads its own work queue +// publishes (design 47 H2 §793, H-A-454). +const publishQueueVerbKey = "seat:" + meshbuilds.Seat + "." + meshbuilds.VerbPublishQueue -// workQueueReader is the builder's work queue local as the versions package's reader (design 47 H2 §793, H-A-448): its items read -// through the tool runtime's stream read in pages, each keyed by its sequence, and its size through the work queue's size; a -// read past workQueueItemsRead is capped. call is stdio.BusState. -func workQueueReader(call func(method string, params any) (json.RawMessage, error), local string) versions.Reader { - none := func() ([]string, error) { return nil, errors.ErrUnsupported } - return versions.Reader{Keys: none, All: func() (versions.StatePage, error) { - out := versions.StatePage{Values: map[string]json.RawMessage{}} - from := uint64(0) - for { - raw, err := call(buscalls.StreamRead, buscalls.StreamReadRequest{Stream: local, From: from, Limit: workQueuePageItems}) - if err != nil { - return out, err - } - var pg buscalls.StreamPage - if err := json.Unmarshal(raw, &pg); err != nil { - return out, fmt.Errorf("the work queue %s answered what is no page of items: %w", local, err) - } - for _, m := range pg.Messages { - out.Values[strconv.FormatUint(m.Sequence, 10)] = m.Body - } - if pg.Next == 0 || len(pg.Messages) == 0 { - return out, nil - } - if len(out.Values) >= workQueueItemsRead { - out.More = true - break - } - from = pg.Next - } - raw, err := call(buscalls.WorkQueueSize, buscalls.WorkQueueRequest{Queue: local}) - var size buscalls.WorkQueueSized - if err == nil { - err = json.Unmarshal(raw, &size) - } +// publishQueueVia asks the seat mesh-builds' verb publish-queue through ask (stdio.Ask). +func publishQueueVia(ask func(key string, body any) (json.RawMessage, error)) func(module, version string) (meshbuilds.PublishQueueAnswer, error) { + return func(module, version string) (meshbuilds.PublishQueueAnswer, error) { + var a meshbuilds.PublishQueueAnswer + raw, err := ask(publishQueueVerbKey, meshbuilds.PublishQueueInput{Module: module, Version: version}) if err != nil { - out.InfoWords = fmt.Sprintf("the work queue's size cannot be read: %v", err) - } else { - out.Keys, out.Bytes = size.Count, size.Bytes + return a, fmt.Errorf("the builder's verb %s did not answer: %w", meshbuilds.VerbPublishQueue, err) } - return out, nil - }} + if err := json.Unmarshal(raw, &a); err != nil { + return a, fmt.Errorf("the verb %s answered what is no queue: %w", meshbuilds.VerbPublishQueue, err) + } + return a, nil + } } // publishActTools are builder_publish_drop and builder_publish_first (design 47 H2 §793): hand acts, recorded in the hand-act log @@ -147,7 +115,7 @@ func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclie // republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act: // it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer // with the versions it queued. -func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { +func republishTools(queue func(module, version string) (meshbuilds.PublishQueueAnswer, error), acted, reasks versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, @@ -182,7 +150,11 @@ func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now fu "module": map[string]any{"type": "string", "description": "list only this module's queued versions, each with its class"}, "version": map[string]any{"type": "string", "description": "with module: only this version, MAJOR.MINOR.PATCH"}}}, Run: func(args map[string]any) (any, error) { - return versions.AnswerQueue(queue, acted, reasks, str(args, "module"), str(args, "version")) + q, err := queue(str(args, "module"), str(args, "version")) + if err != nil { + return nil, err + } + return versions.AnswerQueue(q, acted, reasks) }}, } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go index 673ee5fe4..c2eb35edc 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go @@ -8,8 +8,8 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/buscalls" "mesh-delivery-builder/internal/build" @@ -65,33 +65,25 @@ func TestAStateIsReadInOnePage(t *testing.T) { t.Errorf("health four minutes in = %+v, want healthy", a) } }) - // Design 47 H2 §793, H-A-448: builder_publish_queue reads the work queue publishes through the runtime's stream read, in pages, - // and a read past its bound is capped with the queue's size from the work queue's size; builder_publish_drop and + // Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from + // its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and // builder_publish_first record the hand act first, then write their reasks value. - t.Run("the work queue is read in pages, capped past its bound", func(t *testing.T) { - for _, total := range []int{1200, 9000} { - var methods []string - call := func(method string, params any) (json.RawMessage, error) { - methods = append(methods, method) - if method == buscalls.WorkQueueSize { - return json.Marshal(buscalls.WorkQueueSized{Fill: buscalls.Fill{Count: uint64(total), Bytes: 4096}}) - } - req := params.(buscalls.StreamReadRequest) - var pg buscalls.StreamPage - for n := req.From + 1; n <= uint64(total) && len(pg.Messages) < req.Limit; n++ { - pg.Messages = append(pg.Messages, buscalls.StreamMessage{Sequence: n, Body: json.RawMessage(`{"holder":"oci","class":"again"}`)}) - pg.Next = n - } - if pg.Next == uint64(total) { - pg.Next = 0 - } - return json.Marshal(pg) - } - pg, err := workQueueReader(call, versions.PublishesLocal).All() - capped := total > workQueueItemsRead - if err != nil || pg.More != capped || len(pg.Values) != min(total, workQueueItemsRead) || (capped && pg.Keys != uint64(total)) { - t.Errorf("%d items: read %d, more %v, keys %d (%v) after %v", total, len(pg.Values), pg.More, pg.Keys, err, methods) - } + t.Run("the queue is the builder's verb publish-queue", func(t *testing.T) { + var asked []string + ask := func(key string, body any) (json.RawMessage, error) { + in := body.(meshbuilds.PublishQueueInput) + asked = append(asked, key+" "+in.Module) + return json.Marshal(meshbuilds.PublishQueueAnswer{Total: 9000, Listed: 1, Capped: true, ItemsRead: 4000, QueueItems: 9000, QueueBytes: 4096, + Counts: []meshbuilds.PublishQueueAnswerCountsItem{{Holder: "oci", Class: versions.PublishClassStart, Count: 4000}}, + Queue: []meshbuilds.PublishQueueAnswerQueueItem{{Holder: "oci", Module: "m", Version: "0.1.0", Class: versions.PublishClassStart}}}) + } + none := versions.Reader{Keys: func() ([]string, error) { return nil, nil }} + tools := republishTools(publishQueueVia(ask), none, none, nil, time.Now, meshhandactsclient.Recorder{}) + raw, err := tools[1].Run(map[string]any{"module": "m"}) + a, _ := raw.(versions.Queued) + if err != nil || fmt.Sprint(asked) != "[seat:mesh-builds.publish-queue m]" || a.Total != 9000 || len(a.Queue) != 1 || a.Queue[0].Module != "m" || + len(a.Counts) != 1 || !a.Capped || len(a.Read) != 1 || a.Read[0].KeysRead != 4000 || a.Read[0].StateKeys != 9000 { + t.Errorf("%s asked %v, answered %+v (%v): want the verb asked with the module, its answer carried with its cap", tools[1].Name, asked, a, err) } }) t.Run("drop and first record the hand act, then ask the builder", func(t *testing.T) { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go index 9f5bcb005..f2f59963c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/gone_integration_test.go @@ -103,9 +103,16 @@ func revisionOf(t *testing.T, b *bed, module string) uint64 { return e.Revision() } +// TestAGoneArtifactIsBuiltAgain: 1937/AC3 and AC4, one whole test of the two (design 47 H2 §819: the builder's integration count does +// not rise with the work queue publishes' whole test). +func TestAGoneArtifactIsBuiltAgain(t *testing.T) { + t.Run("a done key whose blob is gone is built again at start", aDoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart) + t.Run("artifacts-gone leads the builder to build the key again", artifactsGoneLeadsTheBuilderToBuildTheKeyAgain) +} + // 1937/AC3 (integration): the builder, started whole, finds the done key of the trunk's fingerprint naming a blob held // answers gone, builds it again as a new revision whose step says so, and leaves the rebuilt key done. -func TestADoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) { +func aDoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) { b := startBed(t) eventsConsumer(t, b, "n1") manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit) @@ -135,7 +142,7 @@ func TestADoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) { // 1937/AC4 (integration): a done key whose blob held answers held at start stays done; when the controller says // artifacts-gone for it, the builder asks again and builds it again. -func TestArtifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) { +func artifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) { b := startBed(t) eventsConsumer(t, b, "n1") manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index f8bdde9b1..0a4b3bdb4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "io" "io/fs" "log" @@ -40,7 +39,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscompose" - "git.novox.be/novox/mesh-sdk/go/membership" "git.novox.be/novox/mesh-sdk/go/operatorask" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" @@ -270,14 +268,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again. acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider} // A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G - // §588): its items on the work queue publishes its requires asks for, named by the bus's binding (design 47 H2 §793, H-A-448). + // §588): its items on the work queue publishes, the module's own stream of its old streams section (design 47 H2 §793, H-A-454). actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf) - publishesBinding := process.NewBinding(meshbroker.Seat, cred.Module, calls.CallKept, logger.Printf, versions.PublishesLocal) - if err := publishesBinding.Register(hub); err != nil { - return fmt.Errorf("hearing the bus's binding of %s: %w", versions.PublishesLocal, err) - } - pq := &publishQueue{js: js, logf: logger.Printf, - binding: func() (membership.BusBinding, error) { return publishesBinding.Names(versions.PublishesLocal) }} + pq := &publishQueue{js: js, logf: logger.Printf, binding: publishesOf(cred.Module)} republish := &versions.Republisher{Reserved: reserved.r, Put: pq.put, Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk), Unreleased: reserved.unreleasedWords, Logf: logger.Printf, @@ -355,10 +348,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("hearing the operator's answers: %w", err) } retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}} - // Once the bus names the work queue: what waits on it is pulled, and the old state's items are put on it once (§793). - publishesBinding.Start() + // What waits on the work queue is pulled, and the old state's items are put on it once (§793). go func() { - publishesBinding.Ask(versions.PublishesLocal) if err := pq.learnAll(ctx); err != nil { logger.Printf("the publishes waiting on the work queue %s are not pulled yet: %v; each holder's are pulled when an item of it is put", versions.PublishesLocal, err) } @@ -531,7 +522,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // published (H-A-328): whether a package built at a fingerprint is published, asked by the runner writing a wait on it. meshbuilds.VerbPublished: publishedVerb(ctx, reserved.r.PublishedOf), // drop (§778; H-A-401, H-A-404): an ask its caller no longer waits for ends could-not-run naming the drop. - meshbuilds.VerbDrop: dropVerb(steps)}) + meshbuilds.VerbDrop: dropVerb(steps), + // publish-queue (§793; H-A-454): the work queue publishes read from its own stream, for builder_publish_queue. + meshbuilds.VerbPublishQueue: publishQueueVerb(ctx, pq)}) if err != nil { return fmt.Errorf("answering the verb reserved: %w", err) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index 69b16fe61..6e9f4c770 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -14,8 +14,8 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-sdk/go/membership" "mesh-delivery-builder/internal/versions" ) @@ -23,12 +23,14 @@ import ( // Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder. // A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and // put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and asks the -// bus module to drop the old state. +// bus module to drop the old state. The seat mesh-builds' verb publish-queue reads the stream: counts per holder and class, the +// oldest and newest, the live item listed first (H-A-454). func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { b := startBed(t) ctx := b.ctx - bind := membership.BusBinding{Local: versions.PublishesLocal, Stream: "mesh-delivery-builder_publishes", Subject: "mesh.bus.mesh-delivery-builder.publishes"} - if _, err := b.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: bind.Stream, Subjects: []string{bind.Subject + ".>"}, + // The stream as the manifest's old streams section declares it: the module's own, retention work-queue, duplicates 24 hours. + bind, _ := publishesOf("mesh-delivery-builder")() + if _, err := b.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: bind.Stream, Subjects: []string{bind.Subject + ".publish.>"}, Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil { t.Fatal(err) } @@ -51,7 +53,7 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { sentCh <- q.Module return versions.SendFate{Sent: true} }} - pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: func() (membership.BusBinding, error) { return bind, nil }} + pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")} item := func(module, class string) versions.QueuedPublish { return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()} } @@ -74,6 +76,12 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup { t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err) } + raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`)) + if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart || + a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap || + a.Queue[0].Module != "live" || a.Capped { + t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err) + } if err := pq.learnAll(ctx); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index 24f658e20..86ffb3e2c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -16,7 +16,10 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/membership" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" @@ -524,14 +527,15 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion { return out } -// publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-448): asked for in -// its requires and named by the bus's binding; each item on `.publish....` under -// the message id `...`. The builder makes the pull consumers at run time as it learns a holder, +// publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-454): a stream of +// the manifest's old streams section with the work-queue retention, the module's own, consumed by the process on the module's +// account; each item on `mesh.stream..event.publish....` under the message id +// `...`. The builder makes the pull consumers at run time as it learns a holder, // `publishes--`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live // class first, at most four sends at once per holder. type publishQueue struct { js jetstream.JetStream - binding func() (membership.BusBinding, error) + binding func() (publishesStream, error) ackWait time.Duration take func(ctx context.Context, m itemMsg) logf func(string, ...any) @@ -540,6 +544,16 @@ type publishQueue struct { pulling map[string]bool } +// publishesStream is where the work queue publishes is on the bus: its stream's name and the subject its items open with. +type publishesStream struct{ Stream, Subject string } + +// publishesOf is module's work queue publishes, its own stream (H-A-454): the stream's bus name and its events' subject. +func publishesOf(module string) func() (publishesStream, error) { + b := publishesStream{Stream: catalogue.StreamBusName(versions.PublishesLocal), + Subject: strings.TrimSuffix(buscompose.ModuleStreamSubject(module, ""), ".")} + return func() (publishesStream, error) { return b, nil } +} + // The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's // largest; the builder ends an item at publish-tries anyway. const ( @@ -553,7 +567,7 @@ func publishesConsumer(holder, class string) string { } // subject is the item's subject on the bus. -func (pq *publishQueue) subject(b membership.BusBinding, q versions.QueuedPublish) string { +func (pq *publishQueue) subject(b publishesStream, q versions.QueuedPublish) string { return b.Subject + "." + versions.PublishEvent(q.Holder, q.Class, q.Module, q.Version) } @@ -744,6 +758,53 @@ func (pq *publishQueue) itemsOf(ctx context.Context, module, version, class stri return out, st, nil } +// publishQueueItemsRead bounds the verb publish-queue's read of the work queue: at most 4,000 items, past them the answer is capped, +// naming the items read and the queue's size (H-A-293). +const publishQueueItemsRead = 4000 + +// reader is the work queue as the versions package's reader, for the seat mesh-builds' verb publish-queue (§793, H-A-454): its +// items read from the stream by sequence, at most publishQueueItemsRead, each keyed by its sequence; a queue holding more is read +// capped, with the stream's message count and bytes. +func (pq *publishQueue) reader(ctx context.Context) versions.Reader { + none := func() ([]string, error) { return nil, errors.ErrUnsupported } + return versions.Reader{Keys: none, All: func() (versions.StatePage, error) { + out := versions.StatePage{Values: map[string]json.RawMessage{}} + b, err := pq.binding() + if err != nil { + return out, err + } + st, err := pq.js.Stream(ctx, b.Stream) + if err != nil { + return out, err + } + info, err := st.Info(ctx) + if err != nil { + return out, err + } + held := int(info.State.Msgs) + if held == 0 { + return out, nil + } + // A work queue takes no second consumer over its subjects, so its items are read by sequence, the deleted passed over. + for seq := info.State.FirstSeq; seq <= info.State.LastSeq && len(out.Values) < publishQueueItemsRead; seq++ { + m, err := st.GetMsg(ctx, seq) + if errors.Is(err, jetstream.ErrMsgNotFound) { + continue // acknowledged or terminated since + } + if err != nil { + return out, err + } + if strings.HasPrefix(m.Subject, b.Subject+".publish.") { + out.Values[strconv.FormatUint(seq, 10)] = json.RawMessage(m.Data) + } + } + if held > len(out.Values) && len(out.Values) >= publishQueueItemsRead { + out.More, out.Keys, out.Bytes = true, info.State.Msgs, info.State.Bytes + } + return out, nil + }} +} + // drop is builder_publish_drop (§793): the version's publish item is ended with who dropped it and why, so no item of it is ever // sent or put again; its items waiting on the work queue are removed, and one held is terminated. func (pq *publishQueue) drop(ctx context.Context, items *publishItems, module, version, by, why string) (string, error) { @@ -886,3 +947,17 @@ func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string, // natsDropTool is the bus module's drop of an entry no accepted manifest asks for (H-A-126), which the old state goes by. const natsDropTool = "nats.nats_drop" + +// publishQueueVerb is the handler of the seat mesh-builds' verb publish-queue (§793; H-A-454): the builder reads its own work queue +// and answers its counts per holder and class, the oldest and newest, and a capped list; builder_publish_queue asks it. +func publishQueueVerb(ctx context.Context, pq *publishQueue) process.SeatHandler { + return func(_ string, body json.RawMessage) (any, error) { + var in meshbuilds.PublishQueueInput + if len(body) > 0 { + if err := json.Unmarshal(body, &in); err != nil { + return nil, fmt.Errorf("the arguments do not fit the verb publish-queue: %w", err) + } + } + return versions.PublishQueueOf(pq.reader(ctx), in.Module, in.Version) + } +} diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index a896cf076..6d3dfad7e 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -5,7 +5,6 @@ go 1.26.0 require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index 14f8280bb..3719c65ce 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) @@ -509,17 +510,20 @@ func PublishItemKey(module, version string) string { return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version) } -// AnswerQueue is builder_publish_queue over the work queue publishes (its items, read in pages through the tool runtime, §793, -// H-A-448) and the two states as the tools bundle reads them (H-A-290 (5)): each read in bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the -// listed queue to one module's (one version's), so an item's class can be read. -func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) { - out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}} +// PublishQueueOf is the seat mesh-builds' verb publish-queue (design 47 H2 §793; H-A-454): the builder's read of its own work queue +// publishes (queue, its items read in bulk, capped past its bound), small at any size of the queue (H-A-290 (5)): the items' count +// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in the order they are sent; module +// (and version) narrow the list to one module's (one version's), so an item's class can be read. +func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQueueAnswer, error) { + out := meshbuilds.PublishQueueAnswer{Counts: []meshbuilds.PublishQueueAnswerCountsItem{}, Queue: []meshbuilds.PublishQueueAnswerQueueItem{}} vals, c, err := queue.values(PublishesLocal) - out.capped(c) + if c != nil { + out.Capped, out.ItemsRead, out.QueueItems, out.QueueBytes = true, int64(c.KeysRead), int64(c.StateKeys), int64(c.StateBytes) + } if err != nil { return out, fmt.Errorf("the work queue %s cannot be read: %w", PublishesLocal, err) } - counts := map[[2]string]*QueueCount{} + counts := map[[2]string]*meshbuilds.PublishQueueAnswerCountsItem{} var listed []QueuedPublish for _, raw := range vals { var q QueuedPublish @@ -532,7 +536,7 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e out.Total++ c := counts[[2]string{q.Holder, q.Class}] if c == nil { - c = &QueueCount{Holder: q.Holder, Class: q.Class, Oldest: q.Queued, Newest: q.Queued} + c = &meshbuilds.PublishQueueAnswerCountsItem{Holder: q.Holder, Class: q.Class, Oldest: q.Queued, Newest: q.Queued} counts[[2]string{q.Holder, q.Class}] = c } c.Count++ @@ -571,10 +575,27 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e if len(out.Queue) == PublishQueueListCap { break } - out.Queue = append(out.Queue, q) + out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version, + Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued}) } } - out.Listed = len(out.Queue) + out.Listed = int64(len(out.Queue)) + return out, nil +} + +// AnswerQueue is builder_publish_queue: the seat mesh-builds' publish-queue answer the builder gave (q, §793, H-A-454) and the two +// states as the tools bundle reads them (H-A-290 (5)), each read in bulk (Reader.All), capped at PublishQueueListCap. +func AnswerQueue(q meshbuilds.PublishQueueAnswer, acted, reasks Reader) (Queued, error) { + out := Queued{Total: int(q.Total), Counts: []QueueCount{}, Queue: []QueuedPublish{}, Listed: int(q.Listed), Acted: map[string]uint64{}, Items: []PublishItem{}} + if q.Capped { + out.capped(&StateCapped{State: PublishesLocal, KeysRead: int(q.ItemsRead), StateKeys: uint64(q.QueueItems), StateBytes: uint64(q.QueueBytes)}) + } + for _, c := range q.Counts { + out.Counts = append(out.Counts, QueueCount{Holder: c.Holder, Class: c.Class, Count: int(c.Count), Oldest: c.Oldest, Newest: c.Newest}) + } + for _, it := range q.Queue { + out.Queue = append(out.Queue, QueuedPublish{V: 1, Holder: it.Holder, Module: it.Module, Version: it.Version, Artifacts: it.Artifacts, Queued: it.Queued, Class: it.Class}) + } avals, c, err := acted.values(ActedState) out.capped(c) if err != nil { diff --git a/modules/mesh-delivery-builder/internal/versions/republish_test.go b/modules/mesh-delivery-builder/internal/versions/republish_test.go index 7485556a7..65fedcfb5 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish_test.go +++ b/modules/mesh-delivery-builder/internal/versions/republish_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) @@ -239,7 +240,7 @@ func TestRepublish(t *testing.T) { return json.RawMessage(`{"mode":"publish","module":"m","version":"0.1.0","tries":3,"next-try":"2026-11-17T01:00:00Z","why":"w"}`), true, nil }} none := Reader{Keys: func() ([]string, error) { return nil, nil }} - answer, err := AnswerQueue(none, none, items, "", "") + answer, err := answerOf(none, none, items, "", "") if err != nil || len(answer.Items) != 1 || answer.Items[0].Tries != 3 || answer.Items[0].Version != "0.1.0" { t.Errorf("builder_publish_queue items = %+v (%v), want the one publish item with its tries", answer.Items, err) } @@ -384,6 +385,24 @@ func TestAQueuedItemIsSentWhileItsBuildStands(t *testing.T) { }) } +// answerOf is builder_publish_queue whole: the builder's verb publish-queue over its work queue (queue), then the tool over the +// verb's answer and the two states (design 47 H2 §793, H-A-454). +func answerOf(queue, acted, reasks Reader, module, version string) (Queued, error) { + q, err := PublishQueueOf(queue, module, version) + if err != nil { + return Queued{}, err + } + raw, err := json.Marshal(q) // the verb's answer travels the bus + if err != nil { + return Queued{}, err + } + var heard meshbuilds.PublishQueueAnswer + if err := json.Unmarshal(raw, &heard); err != nil { + return Queued{}, err + } + return AnswerQueue(heard, acted, reasks) +} + // TestThePublishQueueToolAnswersAtAnySize: design 47 H H-A-290 (5) — builder_publish_queue over two thousand queued versions answers // within a second and small: counts per holder and class with the oldest and newest queue time, a list capped at fifty in the // order they are sent, and one module's item with its class; each state read in bulk where the reader can. @@ -407,7 +426,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) { Get: func(k string) (json.RawMessage, bool, error) { gets++; v, ok := vals[k]; return v, ok, nil }} none := Reader{Keys: func() ([]string, error) { return nil, nil }} start := time.Now() - a, err := AnswerQueue(queue, none, none, "", "") + a, err := answerOf(queue, none, none, "", "") took := time.Since(start) raw, _ := json.Marshal(a) if err != nil || took > time.Second || a.Total != 2000 || len(a.Queue) != PublishQueueListCap || a.Listed != PublishQueueListCap || len(raw) > 64<<10 || gets != 0 { @@ -419,7 +438,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) { if a.Queue[0].Class != PublishClassLive || a.Queue[0].Holder != "go" { t.Errorf("the list opens with %+v, want the go holder's live class, sent first", a.Queue[0]) } - one, err := AnswerQueue(queue, none, none, "m1995", "") + one, err := answerOf(queue, none, none, "m1995", "") if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 { t.Errorf("one module's answer %+v (%v), want its item with its class", one.Queue, err) } @@ -427,12 +446,12 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) { t.Errorf("a state one page holds is answered capped: %+v", a.Read) } queue.All = func() (StatePage, error) { return StatePage{Values: vals, More: true, Keys: 5000, Bytes: 3 << 20}, nil } - if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishesLocal || + if c, err := answerOf(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishesLocal || c.Read[0].KeysRead != 2000 || c.Read[0].StateKeys != 5000 || c.Read[0].StateBytes != 3<<20 || c.Total != 2000 { t.Errorf("a state past its one page answered capped %v, read %+v (%v): want capped, 2000 keys read of 5000, its size said (H-A-293)", c.Capped, c.Read, err) } queue.All = func() (StatePage, error) { return StatePage{}, errors.ErrUnsupported } - if b, err := AnswerQueue(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 { + if b, err := answerOf(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 { t.Errorf("a reader with no bulk read answered %d (%v) with %d single reads, want every key read one by one", b.Total, err, gets) } } diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 73018a52a..c0f8fbea8 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -13,6 +13,8 @@ import ( "testing" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + + "mesh-delivery-builder/internal/versions" ) // The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521, @@ -58,6 +60,9 @@ type manifest struct { MaxAgeDays int `json:"max-age-days"` MaxMiB int `json:"max-mib"` LastPerSubject bool `json:"last-per-subject"` + Retention string `json:"retention"` + Duplicates int `json:"duplicates"` + MaxDeliveries int `json:"max-deliveries"` } `json:"streams"` Logs []struct { Name string `json:"name"` @@ -146,7 +151,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3). if want := []string{"seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions", "seat:git.tag", "seat:git.tags", - "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", + "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions", // A-231: the build asks on each build seat's queue delivery (accept.delivery.). "seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) { @@ -182,9 +187,12 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes. // Design 47 G A-30, §383: and its own findings' events, which its finding writer says. - if len(m.Streams) != 3 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 || + if len(m.Streams) != 4 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 || m.Streams[1].Name != builds.BuildsStream || !reflect.DeepEqual(m.Streams[1].Events, []string{builds.BuildsStreamEvents}) || m.Streams[1].MaxAgeDays != 7 || !m.Streams[1].LastPerSubject || - m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { + m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) || + // Design 47 H2 §793, §820, H-A-446 (2), H-A-454: the work queue publishes in the old section, 64 MiB, 7 days, duplicates 24 h. + m.Streams[3].Name != versions.PublishesLocal || !reflect.DeepEqual(m.Streams[3].Events, []string{"publish.**"}) || m.Streams[3].Retention != "work-queue" || + m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 { t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams) } if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 10 || @@ -202,9 +210,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if len(m.Logs) != 1 || m.Logs[0].Name != builds.BuilderLog || m.Logs[0].MaxAgeDays != builds.BuilderLogDays || m.Logs[0].MaxMiB != builds.BuilderLogMiB { t.Errorf("logs = %+v, want %s kept %d days in %d MiB", m.Logs, builds.BuilderLog, builds.BuilderLogDays, builds.BuilderLogMiB) } - // Design 47 G §52, §369, H-A-328, H2 §778: the seat mesh-builds, held once for the mesh, serving reserved, published and drop. - if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved", "published", "drop"}) { - t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved, published and drop", m.Claims) + // Design 47 G §52, §369, H-A-328, H2 §778, §793: the seat mesh-builds, held once for the mesh, serving reserved, published, drop + // and publish-queue (H-A-454). + if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved", "published", "drop", "publish-queue"}) { + t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved, published, drop and publish-queue", m.Claims) } // A-115: the process answers the verb itself and says so in `serves` beside its process, with no `bus`; the bundle serves no tool named so. var resources []map[string]any @@ -212,8 +221,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Fatal(err) } for _, r := range resources { - if r["type"] == "process" && !reflect.DeepEqual(r["serves"], []any{"seat:mesh-builds.reserved", "seat:mesh-builds.published", "seat:mesh-builds.drop"}) { - t.Errorf("the process serves %v, want seat:mesh-builds.reserved, published and drop (A-115, H-A-328, H2 §778)", r["serves"]) + if r["type"] == "process" && !reflect.DeepEqual(r["serves"], []any{"seat:mesh-builds.reserved", "seat:mesh-builds.published", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue"}) { + t.Errorf("the process serves %v, want seat:mesh-builds.reserved, published, drop and publish-queue (A-115, H-A-328, H2 §778, §793)", r["serves"]) } } if slices.Contains(m.Tools, "reserved") { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index e5e072fb8..b2ddecd54 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -9,7 +9,8 @@ "serves": [ "reserved", "published", - "drop" + "drop", + "publish-queue" ] } ], @@ -95,6 +96,7 @@ "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", + "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", @@ -149,7 +151,8 @@ "streams": [ { "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 }, { "name": "mesh-delivery-builder-builds", "events": ["build.ended.**"], "max-age-days": 7, "max-mib": 16, "last-per-subject": true }, - { "name": "mesh-delivery-builder-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 } + { "name": "mesh-delivery-builder-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 }, + { "name": "publishes", "events": ["publish.**"], "retention": "work-queue", "duplicates": 24, "max-deliveries": 10, "max-age-days": 7, "max-mib": 64 } ], "config": { "toolchain-wait-minutes": { @@ -329,7 +332,8 @@ "serves": [ "seat:mesh-builds.reserved", "seat:mesh-builds.published", - "seat:mesh-builds.drop" + "seat:mesh-builds.drop", + "seat:mesh-builds.publish-queue" ], "run": [ "./mesh-delivery-builder" diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index b6392f0f8..826690715 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -19,13 +19,14 @@ const Form = 1 // whose fields changed while its forms did not (design 47 H2 §780). const ( generatedForms = "1" - generatedFieldsDigest = "AskTakenEvent=6d5d3da7ac39 BuildEntry=3c829a0a3d58 Dependency=90a32b07c563 DropAnswer=7538f2f5e68e PackagePublished=34dd069a6aab PackagePublishedEvent=8e12888c1559 PublishedAnswer=69a801982819 ReservedAnswer=2272a4d6f0f4 ReservedAnswerArtifactsItem=d74eb6e00abb StepAccept=18efeefb6c34 StepEnded=09a07bfbcb15 StepEndedEvent=207787118dac StepRun=dc5893aef7e5 ToolchainAgreed=f3d9ce993734 ToolchainAgreedDigestsItem=1d42893a96b9 ToolchainAgreedEvent=e3502d6f194d VersionReservedEvent=984547d6ad07" + generatedFieldsDigest = "AskTakenEvent=6d5d3da7ac39 BuildEntry=3c829a0a3d58 Dependency=90a32b07c563 DropAnswer=7538f2f5e68e PackagePublished=34dd069a6aab PackagePublishedEvent=8e12888c1559 PublishQueueAnswer=0f1326e35e17 PublishQueueAnswerCountsItem=b770db2f0920 PublishQueueAnswerQueueItem=b1eebc09f41e PublishedAnswer=69a801982819 ReservedAnswer=2272a4d6f0f4 ReservedAnswerArtifactsItem=d74eb6e00abb StepAccept=18efeefb6c34 StepEnded=09a07bfbcb15 StepEndedEvent=207787118dac StepRun=dc5893aef7e5 ToolchainAgreed=f3d9ce993734 ToolchainAgreedDigestsItem=1d42893a96b9 ToolchainAgreedEvent=e3502d6f194d VersionReservedEvent=984547d6ad07" ) const ( - VerbReserved = "reserved" - VerbPublished = "published" - VerbDrop = "drop" + VerbReserved = "reserved" + VerbPublished = "published" + VerbDrop = "drop" + VerbPublishQueue = "publish-queue" ) const ( @@ -42,7 +43,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbReserved, VerbPublished, VerbDrop} + return []string{VerbReserved, VerbPublished, VerbDrop, VerbPublishQueue} } // Events is every event published on the seat's subject: a new copy each call. @@ -289,6 +290,78 @@ type DropInput struct { // DropAnswer is what drop answers. type DropAnswer = map[string]any +// PublishQueueInput is what publish-queue takes. +type PublishQueueInput struct { + // list only this module's items + Module string `json:"module,omitempty"` + // with module: only this version's, MAJOR.MINOR.PATCH + Version string `json:"version,omitempty"` +} + +// PublishQueueAnswerCountsItem is a type of the contract. +type PublishQueueAnswerCountsItem struct { + // the items' class + Class string `json:"class"` + // how many items of the class the holder's queue holds + Count int64 `json:"count"` + // the registry seat's holder + Holder string `json:"holder"` + // when the newest of them was queued + Newest time.Time `json:"newest"` + // when the oldest of them was queued + Oldest time.Time `json:"oldest"` +} + +// The values of PublishQueueAnswerCountsItem's class. +const ( + PublishQueueAnswerCountsItemClassLive = "live" + PublishQueueAnswerCountsItemClassAgain = "again" +) + +// PublishQueueAnswerQueueItem is a type of the contract. +type PublishQueueAnswerQueueItem struct { + // each artifact reference, :, with the digest to publish + Artifacts map[string]string `json:"artifacts,omitempty"` + // live, a build a pipeline's step waits on, pulled first; again, asked again at a start + Class string `json:"class"` + // the registry seat's holder the item is sent to, a module + Holder string `json:"holder"` + // the module whose version is published + Module string `json:"module"` + // when the item was put on the work queue + Queued time.Time `json:"queued"` + // the reserved version, MAJOR.MINOR.PATCH + Version string `json:"version"` +} + +// The values of PublishQueueAnswerQueueItem's class. +const ( + PublishQueueAnswerQueueItemClassLive = "live" + PublishQueueAnswerQueueItemClassAgain = "again" +) + +// PublishQueueAnswer is what publish-queue answers. +type PublishQueueAnswer struct { + // true when the queue held more items than were read; every count is then of the items read + Capped bool `json:"capped,omitempty"` + // per holder and class, sorted by holder, then class + Counts []PublishQueueAnswerCountsItem `json:"counts"` + // is the form of what this value carries, left out while it is 1 (design 47 H2 §780). + Form int64 `json:"form,omitempty"` + // with capped: how many items were read + ItemsRead int64 `json:"items-read,omitempty"` + // how many of total the answer lists + Listed int64 `json:"listed"` + // the first 50 items listed, per holder in the order they are pulled + Queue []PublishQueueAnswerQueueItem `json:"queue"` + // with capped: the queue's size in bytes + QueueBytes int64 `json:"queue-bytes,omitempty"` + // with capped: how many items the queue holds, from its stream's state + QueueItems int64 `json:"queue-items,omitempty"` + // how many items were read + Total int64 `json:"total"` +} + // StepAccept is the body of step. type StepAccept = StepRun @@ -354,6 +427,15 @@ func DecodeDropAnswer(raw []byte) (DropAnswer, int, error) { return v, formRead(raw), nil } +// DecodePublishQueueAnswer reads a PublishQueueAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). +func DecodePublishQueueAnswer(raw []byte) (PublishQueueAnswer, int, error) { + var v PublishQueueAnswer + if err := json.Unmarshal(raw, &v); err != nil { + return v, 0, err + } + return v, formRead(raw), nil +} + // DecodePackagePublishedEvent reads a PackagePublishedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). func DecodePackagePublishedEvent(raw []byte) (PackagePublishedEvent, int, error) { var v PackagePublishedEvent diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index 03b43b7d9..b19a7f796 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -152,6 +152,150 @@ "description": "whether the ask was held here" }, "class": "act" + }, + { + "name": "publish-queue", + "class": "read", + "description": "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size.", + "input": { + "type": "object", + "properties": { + "module": { + "type": "string", + "description": "list only this module's items" + }, + "version": { + "type": "string", + "description": "with module: only this version's, MAJOR.MINOR.PATCH" + } + }, + "required": [] + }, + "answer": { + "type": "object", + "properties": { + "total": { + "type": "integer", + "description": "how many items were read" + }, + "counts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "holder": { + "type": "string", + "description": "the registry seat's holder" + }, + "class": { + "type": "string", + "enum": [ + "live", + "again" + ], + "description": "the items' class" + }, + "count": { + "type": "integer", + "description": "how many items of the class the holder's queue holds" + }, + "oldest": { + "type": "string", + "format": "date-time", + "description": "when the oldest of them was queued" + }, + "newest": { + "type": "string", + "format": "date-time", + "description": "when the newest of them was queued" + } + }, + "required": [ + "holder", + "class", + "count", + "oldest", + "newest" + ] + }, + "description": "per holder and class, sorted by holder, then class" + }, + "queue": { + "type": "array", + "items": { + "type": "object", + "properties": { + "holder": { + "type": "string", + "description": "the registry seat's holder the item is sent to, a module" + }, + "module": { + "type": "string", + "description": "the module whose version is published" + }, + "version": { + "type": "string", + "description": "the reserved version, MAJOR.MINOR.PATCH" + }, + "class": { + "type": "string", + "enum": [ + "live", + "again" + ], + "description": "live, a build a pipeline's step waits on, pulled first; again, asked again at a start" + }, + "artifacts": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "each artifact reference, :, with the digest to publish" + }, + "queued": { + "type": "string", + "format": "date-time", + "description": "when the item was put on the work queue" + } + }, + "required": [ + "holder", + "module", + "version", + "class", + "queued" + ] + }, + "description": "the first 50 items listed, per holder in the order they are pulled" + }, + "listed": { + "type": "integer", + "description": "how many of total the answer lists" + }, + "capped": { + "type": "boolean", + "description": "true when the queue held more items than were read; every count is then of the items read" + }, + "items-read": { + "type": "integer", + "description": "with capped: how many items were read" + }, + "queue-items": { + "type": "integer", + "description": "with capped: how many items the queue holds, from its stream's state" + }, + "queue-bytes": { + "type": "integer", + "description": "with capped: the queue's size in bytes" + } + }, + "required": [ + "total", + "counts", + "queue", + "listed" + ] + } } ], "accepts": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index dc1a4d4c9..bb720d9a6 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -8,6 +8,7 @@ export declare const verbs: { readonly reserved: "reserved"; readonly published: "published"; readonly drop: "drop"; + readonly publishQueue: "publish-queue"; }; export declare const accepts: { @@ -226,6 +227,64 @@ export interface DropInput { /** is what drop answers. */ export type DropAnswer = Record; +/** is what publish-queue takes. */ +export interface PublishQueueInput { + /** list only this module's items */ + "module"?: string; + /** with module: only this version's, MAJOR.MINOR.PATCH */ + "version"?: string; +} + +export interface PublishQueueAnswerCountsItem { + /** the items' class */ + "class": "live" | "again"; + /** how many items of the class the holder's queue holds */ + "count": number; + /** the registry seat's holder */ + "holder": string; + /** when the newest of them was queued */ + "newest": string; + /** when the oldest of them was queued */ + "oldest": string; +} + +export interface PublishQueueAnswerQueueItem { + /** each artifact reference, :, with the digest to publish */ + "artifacts"?: Record; + /** live, a build a pipeline's step waits on, pulled first; again, asked again at a start */ + "class": "live" | "again"; + /** the registry seat's holder the item is sent to, a module */ + "holder": string; + /** the module whose version is published */ + "module": string; + /** when the item was put on the work queue */ + "queued": string; + /** the reserved version, MAJOR.MINOR.PATCH */ + "version": string; +} + +/** is what publish-queue answers. */ +export interface PublishQueueAnswer { + /** true when the queue held more items than were read; every count is then of the items read */ + "capped"?: boolean; + /** per holder and class, sorted by holder, then class */ + "counts": PublishQueueAnswerCountsItem[]; + /** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */ + "form"?: number; + /** with capped: how many items were read */ + "items-read"?: number; + /** how many of total the answer lists */ + "listed": number; + /** the first 50 items listed, per holder in the order they are pulled */ + "queue": PublishQueueAnswerQueueItem[]; + /** with capped: the queue's size in bytes */ + "queue-bytes"?: number; + /** with capped: how many items the queue holds, from its stream's state */ + "queue-items"?: number; + /** how many items were read */ + "total": number; +} + /** is the body of step. */ export type StepAccept = StepRun; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.js b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.js index f454abb13..5b6b56eca 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.js @@ -8,6 +8,7 @@ export const verbs = { reserved: "reserved", published: "published", drop: "drop", + publishQueue: "publish-queue", }; export const accepts = {