From d6d15a5497f6504d0d0aa4d5824b1ccc33f4d179 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:22:59 +0100 Subject: [PATCH] B4: the publish items travel on the work queue publishes, live first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bucket publish-queue listed and reread itself before each send; its items now ride the builder's work queue publishes, pulled per holder and class by consumers the builder makes as it learns a holder, the live class first, a failed send handed back with the pause, an item acknowledged when its publish is heard and terminated when its build is withdrawn, superseded or dropped. builder_publish_drop and builder_publish_first act through reasks after the hand-act log; builder_publish_queue reads the queue through the runtime. The old state's items are moved once at the start, then dropped through the bus module. Builds: design 47 H2 §793, §820; H-A-290, H-A-291, H-A-440, H-A-446 (2), H-A-448 --- .../cmd/mesh-delivery-builder-tools/main.go | 5 +- .../mesh-delivery-builder-tools/republish.go | 91 +++ .../republish_test.go | 59 ++ .../cmd/mesh-delivery-builder/main.go | 36 +- .../publishes_integration_test.go | 172 +++++ .../cmd/mesh-delivery-builder/publishitem.go | 622 +++++++++++++++++- .../mesh-delivery-builder/publishitem_test.go | 109 ++- .../cmd/mesh-delivery-builder/reasks.go | 12 +- .../cmd/mesh-delivery-builder/republish.go | 36 - .../cmd/mesh-delivery-builder/reserve.go | 32 +- .../cmd/mesh-delivery-builder/reserve_test.go | 1 + modules/mesh-delivery-builder/go.mod | 1 + .../internal/build/stuck.go | 21 + .../internal/versions/again.go | 4 + .../internal/versions/republish.go | 366 ++++------- .../internal/versions/republish_test.go | 304 +++------ 16 files changed, 1318 insertions(+), 553 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index a2c0c6c14..cf230b605 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -433,8 +433,9 @@ func main() { tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")), Newest: newestVia(stdio.Ask), Steps: r})) tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) - qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState) - tools = append(tools, republishTools(readerOf(qs), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + as := stdio.State(versions.ActedState) + tools = append(tools, publishActTools(buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + tools = append(tools, republishTools(workQueueReader(stdio.BusState, versions.PublishesLocal), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) ws := stdio.State(build.WaitsState) wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := ws.Get(key) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go index e5dd1e7c4..91c73783d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go @@ -4,11 +4,13 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "strings" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" @@ -53,6 +55,95 @@ func onePage(page func(after string, maxBytes int, budget time.Duration) (stdio. return out, nil } +// workQueuePageItems and workQueueItemsRead bound the read of the work queue publishes: pages of 500 items, at most 4,000 read +// (H-A-293: past them the answer is capped, naming the items read and the queue's size). +const ( + workQueuePageItems = 500 + workQueueItemsRead = 4000 +) + +// workQueueReader is the builder's work queue local as the versions package's reader (design 47 H2 §793, H-A-448): its items read +// through the tool runtime's stream read in pages, each keyed by its sequence, and its size through the work queue's size; a +// read past workQueueItemsRead is capped. call is stdio.BusState. +func workQueueReader(call func(method string, params any) (json.RawMessage, error), local string) versions.Reader { + none := func() ([]string, error) { return nil, errors.ErrUnsupported } + return versions.Reader{Keys: none, All: func() (versions.StatePage, error) { + out := versions.StatePage{Values: map[string]json.RawMessage{}} + from := uint64(0) + for { + raw, err := call(buscalls.StreamRead, buscalls.StreamReadRequest{Stream: local, From: from, Limit: workQueuePageItems}) + if err != nil { + return out, err + } + var pg buscalls.StreamPage + if err := json.Unmarshal(raw, &pg); err != nil { + return out, fmt.Errorf("the work queue %s answered what is no page of items: %w", local, err) + } + for _, m := range pg.Messages { + out.Values[strconv.FormatUint(m.Sequence, 10)] = m.Body + } + if pg.Next == 0 || len(pg.Messages) == 0 { + return out, nil + } + if len(out.Values) >= workQueueItemsRead { + out.More = true + break + } + from = pg.Next + } + raw, err := call(buscalls.WorkQueueSize, buscalls.WorkQueueRequest{Queue: local}) + var size buscalls.WorkQueueSized + if err == nil { + err = json.Unmarshal(raw, &size) + } + if err != nil { + out.InfoWords = fmt.Sprintf("the work queue's size cannot be read: %v", err) + } else { + out.Keys, out.Bytes = size.Count, size.Bytes + } + return out, nil + }} +} + +// publishActTools are builder_publish_drop and builder_publish_first (design 47 H2 §793): hand acts, recorded in the hand-act log +// first, then written to reasks for the builder, which owns the work queue, to perform and answer. +func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + tool := func(name, mode, desc, what string) stdio.Tool { + return stdio.Tool{Name: name, Description: desc, + Input: map[string]any{"type": "object", "required": []string{"module", "version", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the item's module"}, + "version": map[string]any{"type": "string", "description": "the item's version, MAJOR.MINOR.PATCH"}, + "why": map[string]any{"type": "string", "description": "why, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewPublishAct(mode, str(args, "module"), str(args, "version"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("%s %s %s by hand", what, str(args, "module"), str(args, "version")), + Why: str(args, "why"), Cause: name, Verb: mode}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was done: %w", err) + } + raw, err := ask(key, value) + if err != nil { + return nil, err + } + a, err := build.DecodeReAsk(raw) + if err != nil { + return nil, err + } + if a.Refused { + return nil, errors.New(a.Answer) + } + return map[string]any{"key": key, "answer": a.Answer}, nil + }} + } + return []stdio.Tool{ + tool(versions.PublishDropTool, build.ModePublishDrop, versions.PublishDropDescription, "dropped the publish of"), + tool(versions.PublishFirstTool, build.ModePublishFirst, versions.PublishFirstDescription, "put first the publish of"), + } +} + // republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act: // it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer // with the versions it queued. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go index 3f2842f23..673ee5fe4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go @@ -3,10 +3,15 @@ package main import ( "encoding/json" "errors" + "fmt" "testing" "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" + + "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" ) @@ -60,4 +65,58 @@ func TestAStateIsReadInOnePage(t *testing.T) { t.Errorf("health four minutes in = %+v, want healthy", a) } }) + // Design 47 H2 §793, H-A-448: builder_publish_queue reads the work queue publishes through the runtime's stream read, in pages, + // and a read past its bound is capped with the queue's size from the work queue's size; builder_publish_drop and + // builder_publish_first record the hand act first, then write their reasks value. + t.Run("the work queue is read in pages, capped past its bound", func(t *testing.T) { + for _, total := range []int{1200, 9000} { + var methods []string + call := func(method string, params any) (json.RawMessage, error) { + methods = append(methods, method) + if method == buscalls.WorkQueueSize { + return json.Marshal(buscalls.WorkQueueSized{Fill: buscalls.Fill{Count: uint64(total), Bytes: 4096}}) + } + req := params.(buscalls.StreamReadRequest) + var pg buscalls.StreamPage + for n := req.From + 1; n <= uint64(total) && len(pg.Messages) < req.Limit; n++ { + pg.Messages = append(pg.Messages, buscalls.StreamMessage{Sequence: n, Body: json.RawMessage(`{"holder":"oci","class":"again"}`)}) + pg.Next = n + } + if pg.Next == uint64(total) { + pg.Next = 0 + } + return json.Marshal(pg) + } + pg, err := workQueueReader(call, versions.PublishesLocal).All() + capped := total > workQueueItemsRead + if err != nil || pg.More != capped || len(pg.Values) != min(total, workQueueItemsRead) || (capped && pg.Keys != uint64(total)) { + t.Errorf("%d items: read %d, more %v, keys %d (%v) after %v", total, len(pg.Values), pg.More, pg.Keys, err, methods) + } + } + }) + t.Run("drop and first record the hand act, then ask the builder", func(t *testing.T) { + var order []string + ask := func(key string, value []byte) (json.RawMessage, error) { + order = append(order, "ask "+key) + r, _ := build.DecodeReAsk(value) + r.Answer = "done" + return json.Marshal(r) + } + acts := meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) { + order = append(order, "record") + return json.RawMessage(`{"id":"h1"}`), nil + }} + tools := publishActTools(ask, func() time.Time { return time.Unix(5, 0) }, acts) + for _, tool := range tools { + if _, err := tool.Run(map[string]any{"module": "m", "version": "0.1.2", "why": "a test"}); err != nil { + t.Fatalf("%s: %v", tool.Name, err) + } + } + if len(tools) != 2 || fmt.Sprint(order) != "[record ask publish-drop.5000000000 record ask publish-first.5000000000]" { + t.Errorf("tools %d, order %v: want each hand act recorded before its reasks value", len(tools), order) + } + if _, err := tools[0].Run(map[string]any{"module": "m", "why": "no version"}); err == nil { + t.Error("a drop naming no version is not refused") + } + }) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index f50b44e91..f8bdde9b1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "io" "io/fs" "log" @@ -39,6 +40,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/membership" "git.novox.be/novox/mesh-sdk/go/operatorask" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" @@ -268,10 +270,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again. acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider} // A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G - // §588): its states publish-queue and acted, bound once the bus holds them. - queueKV := process.AwaitBucket(ctx, js, conn, own.queue, "no version is published again on an event or by builder_publish_again", logger.Printf) + // §588): its items on the work queue publishes its requires asks for, named by the bus's binding (design 47 H2 §793, H-A-448). actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf) - republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}}, + publishesBinding := process.NewBinding(meshbroker.Seat, cred.Module, calls.CallKept, logger.Printf, versions.PublishesLocal) + if err := publishesBinding.Register(hub); err != nil { + return fmt.Errorf("hearing the bus's binding of %s: %w", versions.PublishesLocal, err) + } + pq := &publishQueue{js: js, logf: logger.Printf, + binding: func() (membership.BusBinding, error) { return publishesBinding.Names(versions.PublishesLocal) }} + republish := &versions.Republisher{Reserved: reserved.r, Put: pq.put, Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk), Unreleased: reserved.unreleasedWords, Logf: logger.Printf, Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a) @@ -300,6 +307,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1)) releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept) if err != nil { return err @@ -318,15 +326,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }, markHeld: reserved.markHeld, moveElsewhere: reserved.movedElsewhere, allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { return reserved.r.All(ctx) }, + send: republish.SendQueued, retired: reserved.r.Retired} // only each module's newest done build is asked at the start (H-A-291 (1)) // A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)). - republish.Ended = items.Done + republish.Ended = items.Ended + pq.take = items.take reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) }) wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept) acts.Republish = republish.Republish + acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) { + return pq.drop(ctx, items, module, version, by, why) + } + acts.PublishFirst = func(ctx context.Context, module, version, _, _ string) (string, error) { + return pq.first(ctx, items, module, version) + } // builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does. acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf) - again := &republishing{rp: republish} // The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel. decided := &decisions{subject: oc.SubjectDecided(cred.Module)} retakeActed := awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}} @@ -340,8 +355,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("hearing the operator's answers: %w", err) } retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}} - queueKV.Ready(again.bound(ctx, js, logger.Printf)) - actedKV.Ready(again.bound(ctx, js, logger.Printf)) + // Once the bus names the work queue: what waits on it is pulled, and the old state's items are put on it once (§793). + publishesBinding.Start() + go func() { + publishesBinding.Ask(versions.PublishesLocal) + if err := pq.learnAll(ctx); err != nil { + logger.Printf("the publishes waiting on the work queue %s are not pulled yet: %v; each holder's are pulled when an item of it is put", versions.PublishesLocal, err) + } + moveOld(ctx, js, own.queue, cred.Module, republish, calls.Call, logger.Printf) + }() errs := make(chan error, 10) b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466) // The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go new file mode 100644 index 000000000..69b16fe61 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -0,0 +1,172 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/membership" + + "mesh-delivery-builder/internal/versions" +) + +// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder. +// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and +// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and asks the +// bus module to drop the old state. +func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { + b := startBed(t) + ctx := b.ctx + bind := membership.BusBinding{Local: versions.PublishesLocal, Stream: "mesh-delivery-builder_publishes", Subject: "mesh.bus.mesh-delivery-builder.publishes"} + if _, err := b.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: bind.Stream, Subjects: []string{bind.Subject + ".>"}, + Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil { + t.Fatal(err) + } + kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} + var mu sync.Mutex + var sent []string + published := map[string]bool{} // the modules whose publish is heard + sentCh := make(chan string, 200) + items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5}, + now: time.Now, logf: t.Logf, + reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) { + mu.Lock() + defer mu.Unlock() + return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil + }, + send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate { + mu.Lock() + sent = append(sent, q.Module) + mu.Unlock() + sentCh <- q.Module + return versions.SendFate{Sent: true} + }} + pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: func() (membership.BusBinding, error) { return bind, nil }} + item := func(module, class string) versions.QueuedPublish { + return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()} + } + away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled + data, _ := json.Marshal(q) + if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil { + t.Fatal(err) + } + } + for i := range 100 { + away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart)) + } + away(item("dropme", versions.PublishClassStart)) + away(item("live", versions.PublishClassLive)) + if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil { + t.Fatalf("drop: %v", err) + } else { + t.Log(answer) + } + if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup { + t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err) + } + if err := pq.learnAll(ctx); err != nil { + t.Fatal(err) + } + select { + case first := <-sentCh: + if first != "live" { + t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first) + } + case <-time.After(30 * time.Second): + t.Fatal("nothing was sent") + } + for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard + select { + case <-sentCh: + case <-time.After(30 * time.Second): + t.Fatal("the again items were not sent") + } + } + deadline := time.After(60 * time.Second) + for { // each publish sent is heard: its item acknowledged, the next sent + mu.Lock() + n, done := len(sent), slices.Clone(sent) + for _, m := range done { + published[m] = true + } + mu.Unlock() + if n >= 101 { + break + } + for _, m := range done { + items.heard(ctx, m, "0.1.0") + } + select { + case <-sentCh: + case <-deadline: + t.Fatalf("sent %d of 101", n) + } + } + mu.Lock() + if slices.Contains(sent, "dropme") || len(sent) != 101 { + t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme")) + } + mu.Unlock() + + // The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked). + old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)}) + if err != nil { + t.Fatal(err) + } + call := func(string, any) (json.RawMessage, error) { + return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) + } + rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, + &findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf) + rv.filled.Store(true) + for _, m := range []string{"m1", "m2", "m3"} { + v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m}) + if err != nil { + t.Fatal(err) + } + if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"}) + if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil { + t.Fatal(err) + } + } + rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) { + data, _ := json.Marshal(q) + ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))) + if err != nil { + return false, err + } + return ack.Duplicate, nil + }, Started: time.Now(), Logf: t.Logf} + var drops []string + drop := func(tool string, body any) (json.RawMessage, error) { + drops = append(drops, fmt.Sprint(tool, body)) + return json.RawMessage(`{}`), nil + } + for range 2 { + moveOld(ctx, b.js, builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState), "mesh-delivery-builder", rp, drop, t.Logf) + } + st, err := b.js.Stream(ctx, bind.Stream) + if err != nil { + t.Fatal(err) + } + info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>")) + if err != nil { + t.Fatal(err) + } + if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 || len(drops) != 2 { + t.Errorf("the move run twice left %v on the work queue, dropped %v: want each of the three once, as again, and the old state dropped through the bus module", info.State.Subjects, drops) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index 652064769..24f658e20 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -7,20 +7,25 @@ import ( "fmt" "io/fs" "os" + "sort" "strconv" "strings" + "sync" "time" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/membership" + "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" ) -// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588): each reservation of a -// done build is a publish item in the builder's re-ask state, the mechanism builder_reserve_done and publish again use. It is -// asked when the build ends and at the builder's start; each try queues the publish and writes the item again with one more try -// and a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue. +// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588; H2 §793): each +// reservation of a done build is a publish item in the builder's re-ask state, asked when the build ends and at the builder's +// start; its try puts the publish on the work queue publishes, and each send counts one try in the item and hands a failed one +// back with a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue. // publishSettings are the settings publish-pause, publish-pause-max and publish-tries. type publishSettings struct { @@ -28,8 +33,8 @@ type publishSettings struct { Tries int } -// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 8 tries. -var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 8} +// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 5 tries (H-A-446 (2)). +var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 5} // pauseAfter is the pause after the try-th try: the setting doubled per try, up to the maximum. func (s publishSettings) pauseAfter(try int) time.Duration { @@ -61,10 +66,15 @@ type publishItems struct { // publish there (H-A-275). Nil does neither. markHeld func(ctx context.Context, v versions.ReservedVersion, held []versions.HeldArtifact) moveElsewhere func(ctx context.Context, v versions.ReservedVersion, j versions.Judged) error - set publishSettings - now func() time.Time - after func(time.Duration) <-chan time.Time - logf func(string, ...any) + // send sends one item pulled from the work queue (Republisher.SendQueued); nil sends none. + send func(ctx context.Context, q versions.QueuedPublish) versions.SendFate + set publishSettings + now func() time.Time + after func(time.Duration) <-chan time.Time + logf func(string, ...any) + + mu sync.Mutex + held map[string][]heldItem // " ": the items sent, their publish not yet heard } func (p *publishItems) wait(ctx context.Context, d time.Duration) bool { @@ -130,8 +140,11 @@ func (p *publishItems) AskAllDone(ctx context.Context) { } } -// Done deletes a version's publish item: its release is written. +// Done deletes a version's publish item and acknowledges its held items: its release is written. func (p *publishItems) Done(ctx context.Context, module, version string) { + for _, h := range p.settle(module, version, func(heldItem) bool { return true }) { + _ = h.m.Ack() + } kv := p.reasks() if kv == nil { return @@ -141,9 +154,10 @@ func (p *publishItems) Done(ctx context.Context, module, version string) { } } -// Try waits for the item's next try, then, while the version is still reserved, queues its publish and writes the item again with -// one more try and the pause after it; once the tries are spent it ends the item loudly. A version no longer reserved (released, -// dropped or taken over) has its item deleted; one whose artifacts are published keeps it until its release is written. +// Try waits for the item's next try, then, while the version is still reserved, puts its publish on the work queue publishes +// under its message id (design 47 H2 §793; H-A-448): an item the queue holds already is put once. It counts no try: each send +// does, and hands its item back with the pause (take). A put that queued nothing is put again after the pause. A version no +// longer reserved (released, dropped or taken over) has its item deleted. func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk) { if !p.wait(ctx, r.NextTry.Sub(p.now())) { return @@ -157,37 +171,215 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err) return } - if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn: never because its artifacts are published (A-456b, H-A-290 (2)) + if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn (A-456b, H-A-290 (2)) p.logf("%s %s: the publish item is ended: %s", r.Module, r.Version, why) _ = kv.Delete(ctx, e.Key()) return } + queued, err := p.republish(ctx, r.Module, r.Version) + if err == nil && len(queued) > 0 { + return + } + r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries)) + p.logf("%s %s: the publish is not on the work queue %s (%v); it is put again after %s", r.Module, r.Version, versions.PublishesLocal, err, p.set.pauseAfter(r.Tries)) + if data, err := json.Marshal(r); err == nil { + if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + p.logf("%s: the publish item could not be written: %v", e.Key(), err) + } + } +} + +// itemMsg is what take reads of an item pulled from the work queue (jetstream.Msg). +type itemMsg interface { + Data() []byte + Ack() error + NakWithDelay(time.Duration) error + Term() error +} + +// take handles one item pulled from the work queue (§793; H-A-440 (2), H-A-448): an item whose publish item ended (dropped, or +// its tries spent) or whose version never publishes is terminated; one whose artifacts are all published is acknowledged; once +// publish-tries sends are spent it is terminated and version-unreleased said; while a published of it may be unheard it is handed +// back with the pause, and one the registries' record confirms is acknowledged (H-A-274); else it is sent, the try counted in its +// publish item, and a send that fails handed back with the pause as its delay, so the queue delivers it again and nothing is put +// again. A sent item is acknowledged when its publish is heard (heard, Done) and delivered again after the ack wait, unheard. +func (p *publishItems) take(ctx context.Context, m itemMsg) { + var q versions.QueuedPublish + if err := json.Unmarshal(m.Data(), &q); err != nil || q.Module == "" || q.Version == "" { + p.logf("an item of the work queue %s that is no publish is terminated: %v", versions.PublishesLocal, err) + _ = m.Term() + return + } + kv := p.reasks() + if kv == nil { + p.logf("%s %s: the re-ask state is not on the bus yet; the item is handed back", q.Module, q.Version) + _ = m.NakWithDelay(p.set.Pause) + return + } + key := versions.PublishItemKey(q.Module, q.Version) + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { // put by builder_publish_again or the move: its tries are counted from now + if err = p.Ask(ctx, q.Module, q.Version); err == nil { + e, err = kv.Get(ctx, key) + } + } + if err != nil { + p.logf("%s %s: its publish item cannot be read (%v); the item is handed back", q.Module, q.Version, err) + _ = m.NakWithDelay(p.set.Pause) + return + } + r, err := build.DecodeReAsk(e.Value()) + if err != nil { + _ = m.Term() + return + } + if r.Answer != "" { + p.logf("%s %s at %s is terminated: its publish item ended: %s", q.Module, q.Version, q.Holder, r.Answer) + _ = m.Term() + return + } + v, found, err := p.reserved(ctx, q.Module, q.Version) + if err != nil { + p.logf("%s %s: its reservation cannot be read (%v); the item is handed back", q.Module, q.Version, err) + _ = m.NakWithDelay(p.set.pauseAfter(r.Tries)) + return + } + if why := versions.NotStanding(v, found); why != "" { + p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, why) + _ = m.Term() + _ = kv.Delete(ctx, key) + return + } + if allPublished(v, q) { + p.logf("%s %s at %s is acknowledged: its artifacts are published", q.Module, q.Version, q.Holder) + _ = m.Ack() + return + } if r.Tries >= p.set.Tries { + _ = m.Term() r.Answer = fmt.Sprintf("the publish of %s %s was tried %d times and the version is not released; builder_publish_again tries it again", r.Module, r.Version, r.Tries) r.Answered = p.now().UTC() - p.logf("%s: %s", e.Key(), r.Answer) + p.logf("%s: %s", key, r.Answer) if p.endedWith != nil { p.endedWith(ctx, v, "the publish ended after "+fmt.Sprint(r.Tries)+" tries") } - } else if why, gone := p.notAgain(ctx, v); gone { - return // moved to another number, whose own item is asked (H-A-275) - } else if why != "" { - // A published of the version may be unheard, or the registry's record ended it: no ask, and no try spent (H-A-272, H-A-274). - p.logf("%s %s: the publish is not asked again: %s; tried again after %s", r.Module, r.Version, why, p.set.pauseAfter(r.Tries)) - r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries)) - } else { - r.Tries++ - if _, err := p.republish(ctx, r.Module, r.Version); err != nil { - p.logf("%s %s: try %d queued no publish: %v", r.Module, r.Version, r.Tries, err) - } - r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries)) + p.write(ctx, kv, key, r, e.Revision()) + return } + if why, gone := p.notAgain(ctx, v); gone { + _ = m.Term() // moved to another number, whose own item is asked (H-A-275) + return + } else if why != "" { + if v2, ok, err := p.reserved(ctx, q.Module, q.Version); err == nil && ok && allPublished(v2, q) { + p.logf("%s %s at %s is acknowledged: %s", q.Module, q.Version, q.Holder, why) + _ = m.Ack() + return + } + p.logf("%s %s: the publish is not sent again: %s; handed back for %s", q.Module, q.Version, why, p.set.pauseAfter(r.Tries)) + _ = m.NakWithDelay(p.set.pauseAfter(r.Tries)) + return + } + fate := versions.SendFate{Failed: "this builder sends no publish"} + if p.send != nil { + fate = p.send(ctx, q) + } + if fate.Ended != "" { + p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, fate.Ended) + _ = m.Term() + return + } + r.Tries++ + r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries)) + p.write(ctx, kv, key, r, e.Revision()) + if fate.Failed != "" { + p.logf("%s %s at %s: try %d sent no publish: %s; handed back for %s", q.Module, q.Version, q.Holder, r.Tries, fate.Failed, p.set.pauseAfter(r.Tries)) + _ = m.NakWithDelay(p.set.pauseAfter(r.Tries)) + return + } + p.hold(q, m) +} + +// write writes a publish item at its revision; a newer write wins. +func (p *publishItems) write(ctx context.Context, kv jetstream.KeyValue, key string, r build.ReAsk, rev uint64) { data, err := json.Marshal(r) if err != nil { return } - if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { - p.logf("%s: the publish item could not be written: %v", e.Key(), err) + if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + p.logf("%s: the publish item could not be written: %v", key, err) + } +} + +// allPublished says none of q's artifacts is missing from v: its publish is heard. +func allPublished(v versions.ReservedVersion, q versions.QueuedPublish) bool { + missing := map[string]bool{} + for _, ref := range v.Missing() { + missing[ref] = true + } + for ref := range q.Artifacts { + if missing[ref] { + return false + } + } + return true +} + +// heldItem is an item sent and not yet acknowledged: its publish is not heard yet. +type heldItem struct { + q versions.QueuedPublish + m itemMsg +} + +// hold keeps a sent item until its publish is heard. +func (p *publishItems) hold(q versions.QueuedPublish, m itemMsg) { + p.mu.Lock() + defer p.mu.Unlock() + if p.held == nil { + p.held = map[string][]heldItem{} + } + k := q.Module + " " + q.Version + p.held[k] = append(p.held[k], heldItem{q: q, m: m}) +} + +// settle takes every held item of module's version that pick picks out of the held ones. +func (p *publishItems) settle(module, version string, pick func(heldItem) bool) []heldItem { + p.mu.Lock() + defer p.mu.Unlock() + k := module + " " + version + var out, keep []heldItem + for _, h := range p.held[k] { + if pick(h) { + out = append(out, h) + } else { + keep = append(keep, h) + } + } + if len(keep) == 0 { + delete(p.held, k) + } else { + p.held[k] = keep + } + return out +} + +// heard acknowledges each held item of module's version whose artifacts are all published now: its publish is heard (§793). +func (p *publishItems) heard(ctx context.Context, module, version string) { + v, found, err := p.reserved(ctx, module, version) + if err != nil || !found { + return + } + for _, h := range p.settle(module, version, func(h heldItem) bool { return allPublished(v, h.q) }) { + p.logf("%s %s at %s is acknowledged: its publish is heard", module, version, h.q.Holder) + _ = h.m.Ack() + } +} + +// Ended deletes a version's publish item and terminates its held items: its build is withdrawn or superseded (H-A-290 (2), +// H-A-291 (1)); an item still on the work queue is terminated when it is pulled. +func (p *publishItems) Ended(ctx context.Context, module, version string) { + p.Done(ctx, module, version) + for _, h := range p.settle(module, version, func(heldItem) bool { return true }) { + _ = h.m.Term() } } @@ -312,10 +504,11 @@ func publishSettingsOf(getenv func(string) string) (publishSettings, error) { return publishSettings{Pause: time.Duration(pause) * time.Second, PauseMax: time.Duration(pmax) * time.Minute, Tries: tries}, nil } -// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b): a done build asks its item, a -// release, a drop and a takeover end it, and a release is recorded in releases, the last kept of each module. +// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b, H2 §793): a done build asks its +// item, a release ends it and acknowledges its held items, a drop and a takeover end it and terminate them, a published heard +// acknowledges those it publishes, and a release is recorded in releases, the last kept of each module. func wirePublishItems(rv *reservations, items *publishItems, releases versions.ReserveStore, kept int) { - rv.built, rv.released = items.Ask, items.Done + rv.built, rv.released, rv.itemEnded, rv.itemHeard = items.Ask, items.Done, items.Ended, items.heard rv.r.ReleasedKept, rv.r.Releases = kept, releases } @@ -330,3 +523,366 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion { } return out } + +// publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-448): asked for in +// its requires and named by the bus's binding; each item on `.publish....` under +// the message id `...`. The builder makes the pull consumers at run time as it learns a holder, +// `publishes--`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live +// class first, at most four sends at once per holder. +type publishQueue struct { + js jetstream.JetStream + binding func() (membership.BusBinding, error) + ackWait time.Duration + take func(ctx context.Context, m itemMsg) + logf func(string, ...any) + + mu sync.Mutex + pulling map[string]bool +} + +// The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's +// largest; the builder ends an item at publish-tries anyway. +const ( + publishesMaxAckPending = versions.PublishSendsAtOnce + publishesMaxDeliver = 10 +) + +// publishesConsumer is the durable consumer of holder's items of class. +func publishesConsumer(holder, class string) string { + return versions.PublishesLocal + "-" + holder + "-" + class +} + +// subject is the item's subject on the bus. +func (pq *publishQueue) subject(b membership.BusBinding, q versions.QueuedPublish) string { + return b.Subject + "." + versions.PublishEvent(q.Holder, q.Class, q.Module, q.Version) +} + +// put puts q on the work queue under its message id; duplicate says the queue holds that id within its window. A put the full +// queue refuses is versions.ErrQueueFull. The item's holder is learnt: its pull starts. +func (pq *publishQueue) put(ctx context.Context, q versions.QueuedPublish) (bool, error) { + b, err := pq.binding() + if err != nil { + return false, err + } + data, err := json.Marshal(q) + if err != nil { + return false, err + } + ack, err := pq.js.PublishMsg(ctx, &nats.Msg{Subject: pq.subject(b, q), Data: data}, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))) + if err != nil { + if w := err.Error(); strings.Contains(w, "maximum bytes") || strings.Contains(w, "maximum messages") || errors.Is(err, jetstream.ErrMaxBytesExceeded) { + return false, fmt.Errorf("%w: %v", versions.ErrQueueFull, err) + } + return false, err + } + pq.learn(ctx, q.Holder) + return ack.Duplicate, nil +} + +// learnAll learns every holder whose items the work queue holds: at the start, so what waits there is pulled. +func (pq *publishQueue) learnAll(ctx context.Context) error { + b, err := pq.binding() + if err != nil { + return err + } + st, err := pq.js.Stream(ctx, b.Stream) + if err != nil { + return err + } + info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.>")) + if err != nil { + return err + } + for subj := range info.State.Subjects { + if holder, _, ok := strings.Cut(strings.TrimPrefix(subj, b.Subject+".publish."), "."); ok { + pq.learn(ctx, holder) // a holder is a module, its name one token + } + } + return nil +} + +// learn starts holder's pull once. +func (pq *publishQueue) learn(ctx context.Context, holder string) { + pq.mu.Lock() + if pq.pulling == nil { + pq.pulling = map[string]bool{} + } + if pq.pulling[holder] { + pq.mu.Unlock() + return + } + pq.pulling[holder] = true + pq.mu.Unlock() + if err := pq.pull(ctx, holder); err != nil { + pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err) + pq.mu.Lock() + delete(pq.pulling, holder) + pq.mu.Unlock() + } +} + +// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item +// waits while a live one waits on the queue. +func (pq *publishQueue) pull(ctx context.Context, holder string) error { + b, err := pq.binding() + if err != nil { + return err + } + chans := map[string]chan jetstream.Msg{} + var live jetstream.Consumer + for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { // the again first: the live is first by the dispatch, not by luck + c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class), + FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy, + AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver}) + if err != nil { + return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err) + } + ch := make(chan jetstream.Msg, publishesMaxAckPending) + cc, err := c.Consume(func(m jetstream.Msg) { ch <- m }, jetstream.PullMaxMessages(publishesMaxAckPending)) + if err != nil { + return fmt.Errorf("the consumer %s is not pulled: %w", publishesConsumer(holder, class), err) + } + go func() { <-ctx.Done(); cc.Stop() }() + chans[class] = ch + if class == versions.PublishClassLive { + live = c + } + } + liveWaits := func() bool { + info, err := live.Info(ctx) + return err == nil && info.NumPending > 0 + } + pq.logf("pulling the publishes of %s, the live first", holder) + go pq.dispatch(ctx, chans[versions.PublishClassLive], chans[versions.PublishClassStart], liveWaits) + return nil +} + +// dispatch runs the items of one holder, at most four at once, every live item before an again one: an again item pulled while +// a live one waits on the queue (liveWaits) runs after it. +func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstream.Msg, liveWaits func() bool) { + sem := make(chan struct{}, versions.PublishSendsAtOnce) + run := func(m jetstream.Msg) { + go func() { + defer func() { <-sem }() + pq.take(ctx, m) + }() + } + var spare jetstream.Msg // an again item pulled while a live one waited + for { + select { + case sem <- struct{}{}: + case <-ctx.Done(): + return + } + select { + case m := <-live: + run(m) + continue + default: + } + m := spare + spare = nil + if m == nil { + select { + case lm := <-live: + run(lm) + continue + case m = <-again: + case <-ctx.Done(): + return + } + } + if liveWaits() { + spare = m + select { + case lm := <-live: + run(lm) + case <-ctx.Done(): + return + } + continue + } + select { + case lm := <-live: // came while the queue was asked + spare = m + run(lm) + default: + run(m) + } + } +} + +// itemsOf are the items the work queue holds of module's version in class (empty for any class), each with its subject. +func (pq *publishQueue) itemsOf(ctx context.Context, module, version, class string) (map[string]versions.QueuedPublish, jetstream.Stream, error) { + b, err := pq.binding() + if err != nil { + return nil, nil, err + } + st, err := pq.js.Stream(ctx, b.Stream) + if err != nil { + return nil, nil, err + } + cls := "*" + if class != "" { + cls = class + } + info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.*."+cls+"."+versions.EscapeToken(module)+"."+versions.EscapeToken(version))) + if err != nil { + return nil, nil, err + } + out := map[string]versions.QueuedPublish{} + for subj := range info.State.Subjects { + raw, err := st.GetLastMsgForSubject(ctx, subj) + if err != nil { + continue + } + var q versions.QueuedPublish + if json.Unmarshal(raw.Data, &q) == nil { + out[subj] = q + } + } + return out, st, nil +} + +// drop is builder_publish_drop (§793): the version's publish item is ended with who dropped it and why, so no item of it is ever +// sent or put again; its items waiting on the work queue are removed, and one held is terminated. +func (pq *publishQueue) drop(ctx context.Context, items *publishItems, module, version, by, why string) (string, error) { + qs, st, err := pq.itemsOf(ctx, module, version, "") + if err != nil { + return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err) + } + if err := items.endItem(ctx, module, version, fmt.Sprintf("dropped by %s: %s", by, why)); err != nil { + return "", err + } + for subj := range qs { + if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil { + return "", fmt.Errorf("the item %s is not terminated: %w", subj, err) + } + } + for _, h := range items.settle(module, version, func(heldItem) bool { return true }) { + _ = h.m.Term() + } + return fmt.Sprintf("the publish of %s %s is dropped: %d items terminated, and none is sent or put again", module, version, len(qs)), nil +} + +// first is builder_publish_first (§793): each again item of the version is put again as live, under the message id +// `.live..`, and the old terminated; a live id the queue holds within its window takes nothing, and the +// old stays, said. +func (pq *publishQueue) first(ctx context.Context, items *publishItems, module, version string) (string, error) { + qs, st, err := pq.itemsOf(ctx, module, version, versions.PublishClassStart) + if err != nil { + return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err) + } + if len(qs) == 0 { + return "", fmt.Errorf("the work queue %s holds no again item of %s %s", versions.PublishesLocal, module, version) + } + var moved, kept []string + for subj, q := range qs { + q.Class = versions.PublishClassLive + dup, err := pq.put(ctx, q) + if err != nil { + return "", fmt.Errorf("%s %s is not put again as live at %s: %w", module, version, q.Holder, err) + } + if dup { + kept = append(kept, q.Holder) + continue + } + if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil { + return "", fmt.Errorf("the again item of %s %s at %s is not terminated: %w", module, version, q.Holder, err) + } + moved = append(moved, q.Holder) + } + for _, h := range items.settle(module, version, func(h heldItem) bool { return h.q.Class == versions.PublishClassStart }) { + _ = h.m.Term() + } + sort.Strings(moved) + answer := fmt.Sprintf("%s %s is put first as live at %s, its again item terminated", module, version, strings.Join(moved, ", ")) + if len(kept) > 0 { + sort.Strings(kept) + answer += fmt.Sprintf("; at %s the work queue holds its live id within the duplicates window, so its again item stays", strings.Join(kept, ", ")) + } + return answer, nil +} + +// endItem writes the version's publish item ended with answer, made when none stands: Try and take then leave it, and a start +// asks it no more. +func (p *publishItems) endItem(ctx context.Context, module, version, answer string) error { + kv := p.reasks() + if kv == nil { + return errors.New("the re-ask state is not on the bus yet") + } + key := versions.PublishItemKey(module, version) + for range 5 { + r := build.ReAsk{V: 1, Why: "the build ended with its version unreleased", By: "the builder", At: p.now().UTC(), Mode: build.ModePublish, Module: module, Version: version} + e, err := kv.Get(ctx, key) + var rev uint64 + switch { + case errors.Is(err, jetstream.ErrKeyNotFound): + case err != nil: + return err + default: + if r, err = build.DecodeReAsk(e.Value()); err != nil { + return err + } + rev = e.Revision() + } + r.Answer, r.Answered = answer, p.now().UTC() + data, err := json.Marshal(r) + if err != nil { + return err + } + if rev == 0 { + _, err = kv.Create(ctx, key, data) + } else { + _, err = kv.Update(ctx, key, data, rev) + } + if err == nil { + return nil + } + if !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return err + } + } + return errors.New("the publish item of " + module + " " + version + " changed five times while it was ended") +} + +// moveOld puts the items of the old state publish-queue on the work queue once at the start (§793): each under its message id, so +// a move run twice puts each once within the duplicates window; then the bucket is dropped through the bus module (its tool +// nats_drop, on the operator's warrant). A builder whose bus holds no such bucket moves nothing. +func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string, rp *versions.Republisher, call toolCaller, logf func(string, ...any)) { + kv, err := js.KeyValue(ctx, bucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return + } + if err != nil { + logf("the old state %s cannot be read, so its items are not moved to the work queue %s: %v", versions.PublishQueueState, versions.PublishesLocal, err) + return + } + keys, err := kv.ListKeys(ctx) + if err != nil { + logf("the old state %s cannot be listed, so its items are not moved: %v", versions.PublishQueueState, err) + return + } + var vals []json.RawMessage + for k := range keys.Keys() { + if e, err := kv.Get(ctx, k); err == nil { + vals = append(vals, json.RawMessage(e.Value())) + } + } + n, err := rp.MoveQueued(ctx, vals) + if err != nil { + logf("the old state %s is not dropped: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err) + return + } + logf("the old state %s: %d of its %d items put on the work queue %s; it is dropped through the bus module", versions.PublishQueueState, n, len(vals), versions.PublishesLocal) + if call == nil { + return + } + if _, err := call(natsDropTool, map[string]any{"module": module, "local": versions.PublishQueueState, + "why": "its items are on the builder's work queue " + versions.PublishesLocal + " (design 47 H2 section 793)"}); err != nil { + logf("the old state %s is not dropped through the bus module: %v", versions.PublishQueueState, err) + } +} + +// natsDropTool is the bus module's drop of an entry no accepted manifest asks for (H-A-126), which the old state goes by. +const natsDropTool = "nats.nats_drop" diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go index 4e2c7ea25..b61891e4b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "maps" "os" "path/filepath" @@ -89,6 +90,29 @@ func (k *itemKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOp return nil } +// itemMsgFake is an item pulled from the work queue: what take settled it with. +type itemMsgFake struct { + data []byte + settled []string // ack, term, nak +} + +func (m *itemMsgFake) Data() []byte { return m.data } +func (m *itemMsgFake) Ack() error { m.settled = append(m.settled, "ack"); return nil } +func (m *itemMsgFake) Term() error { m.settled = append(m.settled, "term"); return nil } +func (m *itemMsgFake) NakWithDelay(d time.Duration) error { + m.settled = append(m.settled, "nak "+d.String()) + return nil +} + +func queuedMsg(t *testing.T, module, version string) *itemMsgFake { + t.Helper() + raw, err := json.Marshal(versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: version, Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: versions.PublishClassLive}) + if err != nil { + t.Fatal(err) + } + return &itemMsgFake{data: raw} +} + func decodeItem(t *testing.T, raw []byte) build.ReAsk { t.Helper() r, err := build.DecodeReAsk(raw) @@ -98,21 +122,21 @@ func decodeItem(t *testing.T, raw []byte) build.ReAsk { return r } -// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a — a version's publish item is asked once, each try queues -// the publish and writes the item again with one more try and the pause doubled up to its maximum, the release deletes it, and -// after the setting's tries it ends with its words, raises the condition and stays; a version no longer reserved has it deleted. +// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a, H2 §793 (H-A-440 (2), H-A-448) — a version's publish +// item is asked once; its try puts the publish on the work queue and counts no try; each send counts one and a failed send hands +// the item back with the pause doubled up to its maximum; after the setting's tries the item is terminated and ends with its words, +// raises the condition and stays; an item whose artifacts are published is acknowledged; the release deletes it; a version no +// longer reserved has it deleted. func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { ctx := context.Background() kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} now := time.Date(2026, 11, 17, 1, 0, 0, 0, time.UTC) - var waited []time.Duration - var queued, ended []string + var queued, ended, sent []string reserved := true v := versions.ReservedVersion{Module: "m", Version: "0.1.2", Artifacts: map[string]string{"m:a": "sha256:a"}, BuiltAt: now} p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: 30 * time.Second, PauseMax: 2 * time.Minute, Tries: 4}, now: func() time.Time { return now }, logf: t.Logf, after: func(d time.Duration) <-chan time.Time { - waited = append(waited, d) now = now.Add(d) c := make(chan time.Time, 1) c <- now @@ -126,7 +150,11 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { }, republish: func(_ context.Context, m, ver string) ([]string, error) { queued = append(queued, m+" "+ver) - return nil, errors.New("no holder") + return []string{m + " " + ver}, nil + }, + send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate { + sent = append(sent, q.Module+" "+q.Version) + return versions.SendFate{Failed: "no holder answers"} }, endedWith: func(_ context.Context, v versions.ReservedVersion, why string) { ended = append(ended, v.Version+": "+why) @@ -135,25 +163,34 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { if err := p.Ask(ctx, "m", "0.1.2"); err != nil || p.Ask(ctx, "m", "0.1.2") != nil || len(kv.vals) != 1 { t.Fatalf("asked: %v, %d items, want one item however often it is asked", err, len(kv.vals)) } - for i := 0; i < 5; i++ { - e, _ := kv.Get(ctx, key) - p.Try(ctx, kv, e, decodeItem(t, e.Value())) + e, _ := kv.Get(ctx, key) + p.Try(ctx, kv, e, decodeItem(t, e.Value())) + if len(queued) != 1 || decodeItem(t, kv.vals[key]).Tries != 0 { + t.Fatalf("the try queued %v, tries %d: want the publish put once, no try counted", queued, decodeItem(t, kv.vals[key]).Tries) + } + var settled []string + for range 5 { + m := queuedMsg(t, "m", "0.1.2") + p.take(ctx, m) + settled = append(settled, m.settled...) } got := decodeItem(t, kv.vals[key]) - if len(queued) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") { - t.Fatalf("queued %v, item %+v, ended %v: want four tries, then the end said with its words and the condition", queued, got, ended) + if len(sent) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") { + t.Fatalf("sent %v, item %+v, ended %v: want four sends, then the end said with its words and the condition", sent, got, ended) } - if want := []time.Duration{30 * time.Second, time.Minute, 2 * time.Minute, 2 * time.Minute}; len(waited) != len(want) || waited[0] != want[0] || waited[1] != want[1] || waited[2] != want[2] || waited[3] != want[3] { - t.Errorf("pauses before the tries and the end = %v, want %v (doubled, capped)", waited, want) + if want := "[nak 30s nak 1m0s nak 2m0s nak 2m0s term]"; fmt.Sprint(settled) != want { + t.Errorf("the items were settled %v, want %s (handed back with the pause doubled and capped, then terminated)", settled, want) } - // An item whose artifacts are all published stays until its release is written, and is tried (A-456b). - queued = nil + m := queuedMsg(t, "m", "0.1.2") + if p.take(ctx, m); fmt.Sprint(m.settled) != "[term]" || len(sent) != 4 { + t.Errorf("an item of an ended publish item was settled %v, sent %d: want it terminated unsent", m.settled, len(sent)) + } + // An item whose artifacts are all published is acknowledged unsent; its publish item stays until its release is written (A-456b). _ = p.Ask(ctx, "m", "0.3.0") pk := versions.PublishItemKey("m", "0.3.0") - pe, _ := kv.Get(ctx, pk) - p.Try(ctx, kv, pe, decodeItem(t, pe.Value())) - if _, ok := kv.vals[pk]; !ok || len(queued) != 1 || decodeItem(t, kv.vals[pk]).Tries != 1 { - t.Errorf("an item whose artifacts are published was deleted or not tried: %v, queued %v", kv.vals, queued) + m = queuedMsg(t, "m", "0.3.0") + if p.take(ctx, m); fmt.Sprint(m.settled) != "[ack]" || len(sent) != 4 || !kv.has(pk) { + t.Errorf("an item whose artifacts are published was settled %v, sent %d, kept %v: want it acknowledged unsent, its publish item kept", m.settled, len(sent), kv.has(pk)) } kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} // At the start every reservation of a done build with no release is asked, one item each. @@ -194,7 +231,7 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { queued = nil _ = p.Ask(ctx, "m", "0.1.2") v.Withdrawn = map[string]string{"0.1.2": "fp"} - e, _ := kv.Get(ctx, key) + e, _ = kv.Get(ctx, key) p.Try(ctx, kv, e, decodeItem(t, e.Value())) if len(kv.vals) != 0 || len(queued) != 0 { t.Errorf("a withdrawn reservation left its item %v, queued %v", kv.vals, queued) @@ -209,7 +246,8 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { } } -// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a — the three settings, 30 s, 30 min and 8 tries unless set, and a +// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a, H2 H-A-446 (2) — the three settings, 30 s, 30 min and 5 tries +// unless set, and a // content that is no number is refused naming its file. func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) { dir := t.TempDir() @@ -222,7 +260,7 @@ func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) { } env := map[string]string{} got, err := publishSettingsOf(func(k string) string { return env[k] }) - if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 8 { + if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 5 { t.Fatalf("defaults = %+v (%v)", got, err) } env["MESH_BUILDER_PUBLISH_PAUSE_FILE"], env["MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE"], env["MESH_BUILDER_PUBLISH_TRIES_FILE"] = write("p", "5\n"), write("m", "2"), write("t", "3") @@ -353,9 +391,12 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) { kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} asked, judged, marked, moved := 0, 0, 0, 0 p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: defaultPublishSettings, now: func() time.Time { return now }, logf: t.Logf, - reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil }, - republish: func(context.Context, string, string) ([]string, error) { asked++; return nil, nil }, - unheard: func(context.Context) (int, error) { return c.unheard, c.countErr }, + reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil }, + send: func(context.Context, versions.QueuedPublish) versions.SendFate { + asked++ + return versions.SendFate{Sent: true} + }, + unheard: func(context.Context) (int, error) { return c.unheard, c.countErr }, judge: func(context.Context, versions.ReservedVersion) (versions.Judged, error) { judged++ return c.judged, nil @@ -374,20 +415,24 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) { t.Fatal(err) } key := versions.PublishItemKey(v.Module, v.Version) - e, _ := kv.Get(ctx, key) - p.Try(ctx, kv, e, decodeItem(t, e.Value())) + m := queuedMsg(t, v.Module, v.Version) + m.data = []byte(strings.ReplaceAll(string(m.data), `"node-tools:a"`, `"node-tools:program"`)) + p.take(ctx, m) if (asked == 1) != c.wantAsked || (marked == 1) != c.wantMark || (moved == 1) != c.wantMov { t.Fatalf("asked %d, marked %d, moved %d; want asked %v, marked %v, moved %v", asked, marked, moved, c.wantAsked, c.wantMark, c.wantMov) } if c.wantMov { - if _, ok := kv.vals[key]; ok { - t.Error("the moved version's old item is written again") + if _, ok := kv.vals[key]; ok || fmt.Sprint(m.settled) != "[term]" { + t.Errorf("the moved version's old item is written again, or its item settled %v, not terminated", m.settled) } return } got := decodeItem(t, kv.vals[key]) - if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || !got.NextTry.After(now) { - t.Errorf("tries %d, next try %s: want a try spent only when asked, and the item written again with its pause", got.Tries, got.NextTry) + if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || (c.wantAsked && !got.NextTry.After(now)) { + t.Errorf("tries %d, next try %s: want a try spent only when sent, the item written with its pause", got.Tries, got.NextTry) + } + if !c.wantAsked && fmt.Sprint(m.settled) != "[nak 30s]" { + t.Errorf("an item not sent was settled %v, want it handed back with the pause and no try spent", m.settled) } if c.unheard >= versions.UnheardBound && judged != 0 { t.Error("the registry is read while a published may be unheard") diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 04990fa20..ca9a050c3 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -65,7 +65,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi released(ctx, b, kv, e, r, logf) return } - if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone { + if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone { // A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch. go versionActed(ctx, acts, kv, e, r, logf) return @@ -208,6 +208,16 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them" } } + } else if r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst { + act := acts.PublishDrop + if r.Mode == build.ModePublishFirst { + act = acts.PublishFirst + } + if act == nil { + err = errors.New("this builder keeps no work queue of publishes") + } else { + r.Answer, err = act(ctx, r.Module, r.Version, r.By, r.Why) + } } else if r.Mode == build.ModeReserveDone { if acts.ReserveDone == nil { err = errors.New("this builder keeps no reservations") diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go index 09aebf982..f53a17617 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go @@ -6,11 +6,8 @@ import ( "encoding/hex" "encoding/json" "fmt" - "sync/atomic" "time" - "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -21,21 +18,6 @@ import ( // The builder's publish again (design 47 G §588, A-73, A-76): the wiring of versions.Republisher to the bus. -// awaitedQueue is the state publish-queue as versions.QueueStore: bound once the bus holds it; its bytes are the cap's. -type awaitedQueue struct{ awaitedReserved } - -func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) { - kv, err := s.kv() - if err != nil { - return 0, err - } - st, err := kv.Status(ctx) - if err != nil { - return 0, err - } - return st.Bytes(), nil -} - // entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry. func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { return func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { @@ -114,21 +96,3 @@ func callWithin(ctx context.Context, call toolCaller, key string, body any) (jso func publishBody(p versions.Publish) gopackageregistry.PublishInput { return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version} } - -// republishing starts the publish again once both its states are bound: the queue is taken up again (design 47 G §588). A version's -// publish is asked when its build ends and at the start (A-457a), never waited on a running or manifest-reads-changed. -type republishing struct { - rp *versions.Republisher - ready atomic.Int32 -} - -func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) func(jetstream.KeyValue) { - return func(jetstream.KeyValue) { - if g.ready.Add(1) != 2 { - return - } - if err := g.rp.Resume(ctx); err != nil { - logf("the queued publishes are not taken up again: %v", err) - } - } -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 358e0679c..76ac4dc12 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -45,12 +45,15 @@ type reservations struct { // built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none. built func(ctx context.Context, module, version string) error released func(ctx context.Context, module, version string) - filled atomic.Bool - bounds func() jetstream.KeyValue - found *findings - bound time.Duration - wake func(context.Context) - logf func(string, ...any) + // ended ends the publish item of a build withdrawn, superseded or dropped, its held items terminated; heard acknowledges the + // held items of a version a published newly marked (design 47 H2 §793). Nil are none. + itemEnded, itemHeard func(ctx context.Context, module, version string) + filled atomic.Bool + bounds func() jetstream.KeyValue + found *findings + bound time.Duration + wake func(context.Context) + logf func(string, ...any) // withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none. withdraw func(module, fingerprint, why string) // call asks a seat's verb: the git seat's tag (design 47 G §370). @@ -146,8 +149,8 @@ func (rv *reservations) took(ctx context.Context, older, newer versions.Reserved } else { rv.logf("%s: the reservation of %s (build of %s, never released) is dropped: the build of %s takes the series over and reserves %s", newer.Module, older.Version, older.Fingerprint, newer.Fingerprint, newer.Version) } - if rv.released != nil { - rv.released(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b) + if rv.itemEnded != nil { + rv.itemEnded(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793) } rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series)) rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module) @@ -350,6 +353,15 @@ func (rv *reservations) published(ctx context.Context, o versions.Outcome, inPro for _, m := range out.Marked { rv.sayPublished(ctx, m, o.Digest) } + if rv.itemHeard != nil { // the publish items a published newly marked are acknowledged (§793) + seen := map[string]bool{} + for _, m := range out.Marked { + if k := m.Reserved.Module + " " + m.Reserved.Version; !seen[k] { + seen[k] = true + rv.itemHeard(ctx, m.Reserved.Module, m.Reserved.Version) + } + } + } } // sayPublished says the seat mesh-builds' package-published of an artifact a published marked. @@ -372,8 +384,8 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str if err != nil { return "", err } - if ok && held && rv.released != nil { - rv.released(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b) + if ok && held && rv.itemEnded != nil { + rv.itemEnded(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793) } if !ok { return "", fmt.Errorf("%s holds no reservation of %s", module, series) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 3573a7b5b..6d744c14d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -457,6 +457,7 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) { var asked, ended []string rv.built = func(_ context.Context, m, v string) error { asked = append(asked, m+" "+v); return nil } rv.released = func(_ context.Context, m, v string) { ended = append(ended, m+" "+v) } + rv.itemEnded = rv.released // a build withdrawn or superseded ends its publish item too (§793) registry := []string{"0.1.37"} rv.r.HeldNumbers = func(context.Context, string, string, []string) ([]string, error) { return registry, nil } rv.filled.Store(true) diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 6d3dfad7e..a896cf076 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -5,6 +5,7 @@ go 1.26.0 require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 2682685e7..4f988dac1 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -74,6 +74,27 @@ const ModePublish = "publish" // optional, name what to publish again; the builder answers with the versions it queued (Queued). const ModeRepublish = "republish" +// ModePublishDrop and ModePublishFirst are the Modes of a reasks value of builder_publish_drop and builder_publish_first (design +// 47 H2 §793): Module and Version name the item; the builder terminates it, or puts it again as live and terminates the old. +const ( + ModePublishDrop = "publish-drop" + ModePublishFirst = "publish-first" +) + +// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) or builder_publish_first (ModePublishFirst) writes +// (§793): its key, `.` (a call is one value), and the value. Module, version and why are required. +func NewPublishAct(mode, module, version, why, by string, at time.Time) (string, []byte, error) { + module, version = strings.TrimSpace(module), strings.TrimSpace(version) + if module == "" || version == "" { + return "", nil, errors.New("give the item's module and version") + } + if strings.TrimSpace(why) == "" { + return "", nil, errors.New("say why, in plain words (why)") + } + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: mode, Module: module, Version: version}) + return mode + "." + strconv.FormatInt(at.UnixNano(), 10), data, err +} + // ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*). const ModeFloorAgain = "floor-again" diff --git a/modules/mesh-delivery-builder/internal/versions/again.go b/modules/mesh-delivery-builder/internal/versions/again.go index 30f11bb42..1f7c13c49 100644 --- a/modules/mesh-delivery-builder/internal/versions/again.go +++ b/modules/mesh-delivery-builder/internal/versions/again.go @@ -27,6 +27,10 @@ type Acts struct { ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error) // DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not. DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error) + // PublishDrop terminates one publish item, and PublishFirst puts it again as live and terminates the old (builder_publish_drop, + // builder_publish_first, design 47 H2 §793): the answer, or why not. + PublishDrop func(ctx context.Context, module, version, by, why string) (string, error) + PublishFirst func(ctx context.Context, module, version, by, why string) (string, error) } // FloorAgain is builder_floor_again of name: refused, since the builder numbers nothing of its own and no claim asks a floor diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index 6e205cc62..14f8280bb 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -17,22 +17,28 @@ import ( // **A reserved version of a done build is published by the builder until its release is written** (design 47 G §588, A-457a): the // publish is an item of the builder's re-ask state, asked when the build ends and at its start, tried with a growing pause and -// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has one queue of -// versions, kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no +// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has its items on +// the builder's work queue `publishes` (design 47 H2 §793), subject `publish....`, pulled by one +// consumer per holder and class, the live class first. A send is the registry seat's `publish` with the digest and no // parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. Nothing waits on a // node-engine `running` or `manifest-reads-changed` any more. const ( - // PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB. + // PublishQueueState is the builder's old state of the queues, whose items are put on the work queue once at the first start + // of this release, then dropped through the bus module (design 47 H2 §793). PublishQueueState = "publish-queue" - PublishQueueMiB = 4 + // PublishesLocal is the builder's work queue of publish items, asked for in its requires (§793, §820; H-A-448). + PublishesLocal = "publishes" // ActedState holds, one key per subject, the last stream sequence the builder acted on; capped at ActedMiB. ActedState = "acted" ActedMiB = 1 // PublishQueueTool is builder_publish_queue. PublishQueueTool = "builder_publish_queue" - // QueueFull is what version-unreleased says of a version the queue's cap refused. + // QueueFull is what version-unreleased says of a version the work queue's cap refused. QueueFull = "queue full" + // PublishDropTool and PublishFirstTool are builder_publish_drop and builder_publish_first (§793). + PublishDropTool = "builder_publish_drop" + PublishFirstTool = "builder_publish_first" // NotStoredWords opens what a refused artifact is said as. NotStoredWords = "not stored: " // VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds @@ -40,11 +46,11 @@ const ( VerbPublish = "publish" VerbVersions = "versions" // PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on; - // PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's queue sends every - // live one before any of the start, each class oldest first (design 47 H H-A-290 (1)). - PublishClassLive = "build-ended-this-run" - PublishClassStart = "asked-again-at-start" - // PublishSendsAtOnce bounds the sends of one holder's queue that run at once (H-A-290 (3)). + // PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled + // live first, each class oldest first (design 47 H H-A-290 (1), H2 §793). + PublishClassLive = "live" + PublishClassStart = "again" + // PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793). PublishSendsAtOnce = 4 // PublishQueueListCap caps the queued versions and the publish items builder_publish_queue lists (H-A-290 (5)). PublishQueueListCap = 50 @@ -77,9 +83,8 @@ func SeatOfArtifact(a meshcatalog.Artifact) (string, bool) { return seat, ok } -// QueuedPublish is the value of a publish-queue key `..` (each token escaped, §73): the -// artifacts of the version this holder publishes, by reference with their digests, when it was queued and whether its -// publish has begun. +// QueuedPublish is one item of the work queue publishes (§793), and the value of a key of the old state publish-queue: the +// artifacts of the version this holder publishes, by reference with their digests, and when it was queued. type QueuedPublish struct { V int `json:"v"` Holder string `json:"holder"` @@ -87,7 +92,6 @@ type QueuedPublish struct { Version string `json:"version"` Artifacts map[string]string `json:"artifacts"` Queued time.Time `json:"queued"` - Begun bool `json:"begun,omitempty"` // Class is PublishClassLive or PublishClassStart (H-A-290 (1)); a value written before it had one is of the start. Class string `json:"class,omitempty"` } @@ -110,25 +114,31 @@ func sortQueued(qs []QueuedPublish) { }) } -// QueueFilter is the filter of every queued version: a queue key has three tokens, which AllFilter (two) never lists on the bus. -const QueueFilter = "*.*.*" - -// QueueKey is the key of holder's queued version of module. +// QueueKey is the key of holder's queued version of module in the old state publish-queue. func QueueKey(holder, module, version string) string { return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version) } +// PublishMsgID is the message id of holder's item of module's version in class (§793): the work queue takes one item of each +// id within its duplicates window, so an item put twice is put once. +func PublishMsgID(holder, class, module, version string) string { + return EscapeToken(holder) + "." + class + "." + EscapeToken(module) + "." + EscapeToken(version) +} + +// PublishEvent is the subject of holder's item of module's version in class within the work queue publishes (§793): +// `publish....`, each token escaped. +func PublishEvent(holder, class, module, version string) string { + return "publish." + PublishMsgID(holder, class, module, version) +} + +// ErrQueueFull is a put the work queue refused as full: said as QueueFull, and asked again at the item's next try (§820). +var ErrQueueFull = errors.New("the work queue " + PublishesLocal + " is full") + // Acted is the value of an acted key: the last stream sequence acted on for its subject. type Acted struct { Sequence uint64 `json:"sequence"` } -// QueueStore is the queue's bucket: compare-and-set, deletion and how many bytes it holds, for its cap. -type QueueStore interface { - ReserveStore - Bytes(ctx context.Context) (uint64, error) -} - // Publish is one artifact's publish again: what the registry seat's publish takes, without parts. type Publish struct { Seat, Kind, Name, Version, Digest, Fingerprint, Commit, Transfer string @@ -137,7 +147,9 @@ type Publish struct { // Republisher publishes reserved versions again (design 47 G §588). type Republisher struct { Reserved *Reserver - Queue QueueStore + // Put puts one item on the work queue publishes under its message id (PublishMsgID): duplicate says the queue holds that + // id within its duplicates window and took nothing; a full queue is ErrQueueFull. + Put func(ctx context.Context, q QueuedPublish) (duplicate bool, err error) // Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry). Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) // Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words. @@ -146,8 +158,6 @@ type Republisher struct { Unreleased func(ctx context.Context, v ReservedVersion, why string) Now func() time.Time Logf func(string, ...any) - // Go starts the sending of a holder's queue; nil starts a goroutine. - Go func(func()) // Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat // of the assignments takes too, read from the builder's start (design 47 G A-416, A-416a); nil answers none. Holders func(seat string) []string @@ -158,11 +168,7 @@ type Republisher struct { // nil ends none. Ended func(ctx context.Context, module, version string) - mu sync.Mutex - busy map[string]bool - // relist marks a holder whose pass lists again before its next start-class send: a live publish was queued meanwhile - // (H-A-291 (2)). - relist map[string]bool + mu sync.Mutex // lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock, // or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last. lastQueued time.Time @@ -292,18 +298,9 @@ func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[stri return out, nil } -// drop deletes every queued key of module's version and ends its publish item: it never publishes (H-A-290 (2)), said with why. +// drop ends module's version's publish item: it never publishes (H-A-290 (2)), said with why; its items on the work queue are +// terminated when they are pulled (§793). func (r *Republisher) drop(ctx context.Context, module, version, why string) { - ctx = context.WithoutCancel(ctx) - keys, err := r.Queue.KeysOf(ctx, "*."+EscapeToken(module)+"."+EscapeToken(version)) - if err != nil { - r.logf("publish again: %s %s never publishes (%s), and its queued keys cannot be listed: %v", module, version, why, err) - } - for _, k := range keys { - if err := r.Queue.Delete(ctx, k); err != nil { - r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err) - } - } r.end(ctx, module, version, why) } @@ -338,13 +335,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf sort.SliceStable(vs, func(i, j int) bool { return vs[i].At.After(vs[j].At) }) var queued []string var errs []error - started := map[string]bool{} - // Every holder given a version is drained, whatever a later version met: a version queued and never drained waits forever. - defer func() { - for holder := range started { - r.drainLater(ctx, holder) - } - }() for _, v := range vs { if max > 0 && len(queued) >= max { break @@ -396,7 +386,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf return queued, err } did = did || ok - started[holder] = true } if did { queued = append(queued, v.Module+" "+v.Version) @@ -405,42 +394,25 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf return queued, errors.Join(errs...) } -// enqueue puts one holder's artifacts of a version in its queue: not again when it is queued already, and not past the -// section's cap (said as queue full). +// enqueue puts one holder's artifacts of a version on the work queue under its message id: not again while the queue holds that +// id (its duplicates window), which stands queued, and not past the queue's cap (said as queue full). func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder string, arts map[string]string) (bool, error) { - key := QueueKey(holder, v.Module, v.Version) - if _, _, found, err := r.Queue.Get(ctx, key); err != nil { - return false, err - } else if found { - r.logf("publish again: %s %s is queued at %s already", v.Module, v.Version, holder) - return false, nil - } - raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)}) - used, err := r.Queue.Bytes(ctx) - if err != nil { - return false, fmt.Errorf("the size of the state %s cannot be read: %w", PublishQueueState, err) - } - if used+uint64(len(raw)) > PublishQueueMiB<<20 { + q := QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)} + dup, err := r.Put(ctx, q) + switch { + case errors.Is(err, ErrQueueFull): + r.logf("publish again: %s %s is not queued at %s: %v; its publish item asks it again at its next try", v.Module, v.Version, holder, err) if r.Unreleased != nil { r.Unreleased(ctx, v, QueueFull) } return false, nil + case err != nil: + return false, fmt.Errorf("%s %s cannot be queued at %s: %w", v.Module, v.Version, holder, err) + case dup: + r.logf("publish again: %s %s is queued at %s already (class %s): the work queue holds its message id", v.Module, v.Version, holder, q.Class) + return true, nil } - if _, err := r.Queue.Create(ctx, key, raw); err != nil { - if errors.Is(err, ErrConflict) { - return false, nil - } - return false, fmt.Errorf("%s cannot be queued: %w", key, err) - } - r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), r.classOf(v)) - if r.classOf(v) == PublishClassLive { // a pass sending a start list lists again, so this one goes next (H-A-291 (2)) - r.mu.Lock() - if r.relist == nil { - r.relist = map[string]bool{} - } - r.relist[holder] = true - r.mu.Unlock() - } + r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), q.Class) return true, nil } @@ -537,15 +509,15 @@ func PublishItemKey(module, version string) string { return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version) } -// AnswerQueue is builder_publish_queue over the three states as the tools bundle reads them (H-A-290 (5)): each state read in -// bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the +// AnswerQueue is builder_publish_queue over the work queue publishes (its items, read in pages through the tool runtime, §793, +// H-A-448) and the two states as the tools bundle reads them (H-A-290 (5)): each read in bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the // listed queue to one module's (one version's), so an item's class can be read. func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) { out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}} - vals, c, err := queue.values(PublishQueueState) + vals, c, err := queue.values(PublishesLocal) out.capped(c) if err != nil { - return out, fmt.Errorf("the state %s cannot be read: %w", PublishQueueState, err) + return out, fmt.Errorf("the work queue %s cannot be read: %w", PublishesLocal, err) } counts := map[[2]string]*QueueCount{} var listed []QueuedPublish @@ -637,13 +609,21 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e } // PublishQueueDescription is builder_publish_queue's description. -const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588, H H-A-290): " + - "how many each registry seat holder's queue holds per class (build-ended-this-run, sent first, or asked-again-at-start) with the oldest and newest " + - "queue time; the first 50 in the order they are sent, each with its class, its artifacts' digests, when it was queued and whether its publish has " + - "begun (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " + +const PublishQueueDescription = "The versions the builder will publish again (its work queue " + PublishesLocal + ", design 47 G section 588, H2 section 793): " + + "how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, pulled first, or again, asked again at a start) " + + "with the oldest and newest queue time; the first 50 in the order they are pulled, each with its class, its artifacts' digests and when it was " + + "queued (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " + "sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " + - "tries and next try, and why one ended after its tries. Each state is read in one page; a state past it is answered capped, naming the " + - "keys read and the state's size (H-A-293). Read-only." + "tries and next try, and why one ended (its tries spent, or dropped). A read past its pages is answered capped, naming the items read and the " + + "queue's size (H-A-293). Read-only." + +// PublishDropDescription and PublishFirstDescription are builder_publish_drop's and builder_publish_first's (§793). +const PublishDropDescription = "Drop one version's publish (design 47 H2 section 793): its items on the work queue " + PublishesLocal + " are " + + "terminated at every holder, and its publish item ended with who dropped it and why, so it is never sent or put again; builder_publish_again " + + "asks it anew. A hand act, recorded in the hand-act log first." +const PublishFirstDescription = "Put one version's publish first (design 47 H2 section 793): each of its again items on the work queue " + + PublishesLocal + " is put again as live, under the message id .live.., pulled before every again item, and the old " + + "terminated. A hand act, recorded in the hand-act log first." // PublishAgainDescription is builder_publish_again's description. const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " + @@ -652,17 +632,14 @@ const PublishAgainDescription = "Publish reserved versions again (design 47 G se "with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " + "recorded in the hand-act log." -// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588): a version whose reservation -// was withdrawn or taken over is dropped with its publish item ended (H-A-290 (2)), and one of the live class whose build ended -// before this run is of the start's now (H-A-290 (1)). It reads the reservations once. -func (r *Republisher) Resume(ctx context.Context) error { - keys, err := r.Queue.KeysOf(ctx, QueueFilter) - if err != nil { - return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err) - } +// MoveQueued puts the values of the old state publish-queue on the work queue, once (§793): each under its message id, so a +// move run twice puts each once within the duplicates window; its class as of this run (a build ended before it is of the +// start's, H-A-290 (1)); a version whose reservation was withdrawn, taken over or superseded is not put, its publish item +// ended (H-A-290 (2), H-A-291 (1)). It reads the reservations once, and answers how many it put. +func (r *Republisher) MoveQueued(ctx context.Context, vals []json.RawMessage) (int, error) { all, err := r.Reserved.All(ctx) if err != nil { - return fmt.Errorf("the queued publishes are not taken up: %w", err) + return 0, fmt.Errorf("the queued publishes are not moved: %w", err) } reserved := make(map[string]ReservedVersion, len(all)) for _, v := range all { @@ -670,16 +647,12 @@ func (r *Republisher) Resume(ctx context.Context) error { } retired, err := r.Reserved.Retired(ctx, all) if err != nil { - r.logf("publish again: the superseded builds are not retired at the start: %v", err) + r.logf("publish again: the superseded builds are not retired at the move: %v", err) } - holders := map[string]bool{} - for _, k := range keys { - raw, rev, found, err := r.Queue.Get(ctx, k) - if err != nil || !found { - continue - } + put := 0 + for _, raw := range vals { var q QueuedPublish - if json.Unmarshal(raw, &q) != nil { + if json.Unmarshal(raw, &q) != nil || q.Holder == "" { continue } v, ok := reserved[q.Module+" "+q.Version] @@ -688,126 +661,38 @@ func (r *Republisher) Resume(ctx context.Context) error { why = retired[q.Module+" "+q.Version] } if why != "" { - if err := r.Queue.Delete(ctx, k); err != nil { - r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err) - } r.end(ctx, q.Module, q.Version, why) continue } - if class := r.classOf(v); q.Class != class { - q.Class = class - if raw, err := json.Marshal(q); err == nil { - if _, err := r.Queue.Update(ctx, k, raw, rev); err != nil { - r.logf("%s keeps its class %s at the start: %v", k, q.Class, err) - } - } - } - holders[q.Holder] = true - } - for h := range holders { - r.drainLater(ctx, h) - } - return nil -} - -func (r *Republisher) drainLater(ctx context.Context, holder string) { - r.mu.Lock() - if r.busy == nil { - r.busy = map[string]bool{} - } - if r.busy[holder] { - r.mu.Unlock() - return - } - r.busy[holder] = true - r.mu.Unlock() - run := func() { - defer func() { r.mu.Lock(); delete(r.busy, holder); r.mu.Unlock() }() - r.Drain(ctx, holder) - } - if r.Go != nil { - r.Go(run) - return - } - go run() -} - -// Drain sends holder's queued versions in passes (H-A-290 (3)): a pass lists the holder's keys once, reads each once, and sends -// them in their order (the live class first, each class oldest first), PublishSendsAtOnce at a time; the keys are listed again -// only when the pass is spent, or when a live publish was queued while it sends a start list: then the sends in flight end and -// the keys are listed again, so the live one goes next (H-A-291 (2)); a start-class one queued meanwhile waits for the pass's end. -// Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said. -func (r *Republisher) Drain(ctx context.Context, holder string) { - for ctx.Err() == nil { - r.mu.Lock() - delete(r.relist, holder) - r.mu.Unlock() - keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*") + q.Class = r.classOf(v) + dup, err := r.Put(ctx, q) if err != nil { - r.logf("the queue of %s cannot be listed: %v", holder, err) - return + return put, fmt.Errorf("%s %s is not moved to the work queue %s: %w", q.Module, q.Version, PublishesLocal, err) } - items := make([]QueuedPublish, 0, len(keys)) - for _, k := range keys { - if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found { - var q QueuedPublish - if json.Unmarshal(raw, &q) == nil { - items = append(items, q) - } - } + if !dup { + put++ } - if len(items) == 0 { - return - } - sortQueued(items) - sem := make(chan struct{}, PublishSendsAtOnce) - var wg sync.WaitGroup - for _, q := range items { - if ctx.Err() != nil { - break - } - sem <- struct{}{} - r.mu.Lock() - again := q.Class != PublishClassLive && r.relist[holder] - r.mu.Unlock() - if again { - <-sem - r.logf("publish again: %s's pass lists again after its sends in flight: a live publish was queued", holder) - break - } - wg.Add(1) - go func() { - defer func() { <-sem; wg.Done() }() - r.send(ctx, q) - }() - } - wg.Wait() } + return put, nil } -// send publishes one queued version's artifacts and says the outcome, then deletes its key. Every call it makes honours the one -// bound it is given, UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why -// (H-A-290 (4)). -func (r *Republisher) send(ctx context.Context, q QueuedPublish) { - key := QueueKey(q.Holder, q.Module, q.Version) - defer func() { - if err := r.Queue.Delete(context.WithoutCancel(ctx), key); err != nil { - r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err) - } - }() +// SendFate is what SendQueued did with one item (§793): Sent, every artifact sent (its item acknowledged when its publish is +// heard or confirmed); Ended, why the version never publishes (its item terminated); Failed, the holder's or the call's words +// (its item handed back with the pause). +type SendFate struct { + Sent bool + Ended, Failed string +} + +// SendQueued publishes one item's artifacts and says the outcome. Every call it makes honours the one bound it is given, +// UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why (H-A-290 (4)). +func (r *Republisher) SendQueued(ctx context.Context, q QueuedPublish) SendFate { bound, cancel := context.WithTimeout(ctx, UnreleasedBound) defer cancel() - if !q.Begun { - q.Begun = true - raw, _ := json.Marshal(q) - if _, rev, found, err := r.Queue.Get(bound, key); err == nil && found { - _, _ = r.Queue.Update(bound, key, raw, rev) - } - } all, err := r.Reserved.All(bound) if err != nil { - r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; its publish item asks it again", q.Module, q.Version, q.Holder, err) - return + r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; it is handed back", q.Module, q.Version, q.Holder, err) + return SendFate{Failed: "its reservation cannot be read: " + err.Error()} } var v ReservedVersion found := false @@ -830,43 +715,64 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) { } if why != "" { r.end(ctx, q.Module, q.Version, why) - return + return SendFate{Ended: why} } e, err := r.Entry(bound, v.Fingerprint) if err != nil { - r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err)) - return + words := fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err) + r.say(ctx, v, words) + return SendFate{Failed: words} + } + missing := map[string]bool{} + for _, ref := range v.Missing() { + missing[ref] = true } refs := make([]string, 0, len(q.Artifacts)) for ref := range q.Artifacts { + if _, no := v.Unstored[ref]; no || !missing[ref] { + continue // published already, or refused as not stored: said once, never sent again + } refs = append(refs, ref) } sort.Strings(refs) + if len(refs) == 0 { + return SendFate{Ended: "none of its artifacts is left to send: each is published or refused as not stored"} + } + var failed []string + sent := 0 for _, ref := range refs { seat, name, kind, ok := publishTarget(e, q.Module, ref) if !ok { r.say(ctx, v, ref+": the catalog entry names no registry seat for it") - return + return SendFate{Failed: ref + ": the catalog entry names no registry seat for it"} } err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint, Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))}) if err == nil { r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref]) + sent++ + continue } - if err != nil { - r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err) - words := err.Error() - if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) { - if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil { - r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr) - } else if ok { - r.say(ctx, v, ref+" "+NotStoredWords+words) - } - continue + r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err) + words := err.Error() + if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) { + if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil { + r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr) + } else if ok { + r.say(ctx, v, ref+" "+NotStoredWords+words) } - r.say(ctx, v, ref+": "+words) + continue } + r.say(ctx, v, ref+": "+words) + failed = append(failed, ref+": "+words) } + switch { + case len(failed) > 0: + return SendFate{Failed: strings.Join(failed, "; ")} + case sent == 0: + return SendFate{Ended: "each artifact left was refused as not stored"} + } + return SendFate{Sent: true} } // publishTarget is the registry seat, the name and the kind module's artifact ref is published as, from its catalog entry; ok is diff --git a/modules/mesh-delivery-builder/internal/versions/republish_test.go b/modules/mesh-delivery-builder/internal/versions/republish_test.go index 51d1256a4..7485556a7 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish_test.go +++ b/modules/mesh-delivery-builder/internal/versions/republish_test.go @@ -9,51 +9,40 @@ import ( "slices" "strings" "sync" - "sync/atomic" "testing" - "testing/synctest" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) -type queueMem struct { - *reserveMem - bytes uint64 - // listings counts the listings of one holder's keys (a filter `.*.*`), what a drain's pass lists. - listings atomic.Int32 -} - -func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil } - -func (q *queueMem) KeysOf(ctx context.Context, filter string) ([]string, error) { - if strings.HasSuffix(filter, ".*.*") && filter != QueueFilter { - q.listings.Add(1) - } - return q.reserveMem.KeysOf(ctx, filter) -} - // republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by -// the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record. +// the Go seat), one holder module `oci` that claims the oci seat, a work queue that records its puts (deduplicating by message +// id when window is set, refusing when full) and sends each put at once as its pull would, and a Send and a Unreleased that +// record. type republishBed struct { *Republisher reserved *reserveMem - queue *queueMem now time.Time - mu sync.Mutex // sent and said, which a holder's sends at once write + mu sync.Mutex // sent and said, which sends at once write sent []Publish said []string ended []string refuse string // the words Send fails with, empty for none // holders are the registry seats' holders the seat-holders view names (design 47 G A-416). holders map[string][]string + // puts are the items the work queue took; window deduplicates by message id; full refuses every put; held keeps the puts + // unsent. + puts []QueuedPublish + ids map[string]bool + window, full bool + held bool } func newRepublishBed(t *testing.T) *republishBed { t.Helper() - b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)} + b := &republishBed{reserved: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC), ids: map[string]bool{}} res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }} - b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Now: func() time.Time { return b.now }, Go: func(f func()) { f() }, + b.Republisher = &Republisher{Reserved: res, Now: func() time.Time { return b.now }, Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) { if fp == "src2:holder" { return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil @@ -83,6 +72,21 @@ func newRepublishBed(t *testing.T) *republishBed { b.ended = append(b.ended, module+" "+version) }, Logf: t.Logf} + b.Put = func(ctx context.Context, q QueuedPublish) (bool, error) { + if b.full { + return false, fmt.Errorf("%w: maximum bytes exceeded", ErrQueueFull) + } + id := PublishMsgID(q.Holder, q.Class, q.Module, q.Version) + if b.window && b.ids[id] { + return true, nil + } + b.ids[id] = true + b.puts = append(b.puts, q) + if !b.held { + b.SendQueued(ctx, q) + } + return false, nil + } b.Holders = func(seat string) []string { return b.holders[seat] } return b } @@ -125,8 +129,8 @@ func TestRepublish(t *testing.T) { p.Transfer != "republish-m-"+strings.ReplaceAll(v.Version, ".", "-")+"-prog" || strings.ContainsAny(p.Transfer, "/. *>") { t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p) } - if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 { - t.Errorf("a key stays in the queue after its outcome: %v", keys) + if len(b.puts) != 1 || b.puts[0].Holder != "oci" || b.puts[0].Class != PublishClassLive { + t.Errorf("the work queue took %+v, want one live item at oci", b.puts) } }) t.Run("an entry that cannot be read stops neither the other versions nor the drain", func(t *testing.T) { @@ -173,22 +177,42 @@ func TestRepublish(t *testing.T) { t.Run("a queue past its cap says queue full", func(t *testing.T) { b := newRepublishBed(t) b.reserve(t, "m") - b.queue.bytes = PublishQueueMiB << 20 + b.full = true b.holders = map[string][]string{SeatArtifacts: {"oci"}} _, _ = b.Republish(ctx, "", "") if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) { t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said) } }) - t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) { + t.Run("an item put twice within the duplicates window is put once, and stands queued", func(t *testing.T) { b := newRepublishBed(t) v := b.reserve(t, "m") - raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true}) - if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil { + b.window, b.held = true, true + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + q1, _ := b.Republish(ctx, "m", v.Version) + q2, _ := b.Republish(ctx, "m", v.Version) + if len(b.puts) != 1 || len(q1) != 1 || len(q2) != 1 { + t.Errorf("puts %d, queued %v then %v: want one item, queued both times", len(b.puts), q1, q2) + } + }) + t.Run("the move puts each item of the old state once, its class as of this run, a superseded one not", func(t *testing.T) { + b := newRepublishBed(t) + v := b.reserve(t, "m") + w := b.reserve(t, "n") + b.Started = b.now + b.window, b.held = true, true + raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Class: "build-ended-this-run"}) + gone, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "n", Version: w.Version, Artifacts: map[string]string{"n:prog": "sha256:aa"}, Queued: b.now}) + if _, _, err := b.Reserved.Withdraw(ctx, "n", w.Version); err != nil { t.Fatal(err) } - if err := b.Resume(ctx); err != nil || len(b.sent) != 1 { - t.Errorf("resume sent %d (%v), want the begun version sent anew", len(b.sent), err) + for range 2 { + if _, err := b.MoveQueued(ctx, []json.RawMessage{raw, gone}); err != nil { + t.Fatal(err) + } + } + if len(b.puts) != 1 || b.puts[0].Class != PublishClassStart || fmt.Sprint(b.ended) != "[n "+w.Version+" n "+w.Version+"]" { + t.Errorf("puts %+v, ended %v: want m once as again (its build ended before this run), n's item ended", b.puts, b.ended) } }) t.Run("after a fresh start the tool queues at the seat's holder, no running heard", func(t *testing.T) { @@ -262,98 +286,31 @@ func TestTheRegistriesRecordSaysWhetherAVersionIsHeld(t *testing.T) { } // queueAt writes a queued version of module m at holder oci directly, as a start finds it. -func (b *republishBed) queueAt(t *testing.T, v ReservedVersion, class string, at time.Time) { - t.Helper() - raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class}) - if _, err := b.queue.Create(context.Background(), QueueKey("oci", v.Module, v.Version), raw); err != nil { - t.Fatal(err) - } +func (b *republishBed) queueAt(v ReservedVersion, class string, at time.Time) QueuedPublish { + return QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class} } -// TestAHoldersQueueSendsLiveFirstInPasses: design 47 H H-A-290 (1)-(3) — a publish whose build ended in this run is sent before -// any of a build that ended before the start, each class oldest first; a pass lists the holder's keys once and sends them, at most -// four at a time; a build whose reservation was withdrawn or taken over never publishes, its key dropped and its publish item ended, -// at enqueue, at the pass and at the start. -func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) { +// TestAQueuedItemIsSentWhileItsBuildStands: design 47 H H-A-290 (2), H-A-291 (1), H2 §793 — an item is put at the holder in the +// class of its build (live when it ended in this run); a build whose reservation was withdrawn, taken over or superseded by a newer +// patch or release of its series never publishes, its publish item ended, at the put and at the send; an item whose artifacts are +// all published or refused as not stored ends unsent. +func TestAQueuedItemIsSentWhileItsBuildStands(t *testing.T) { ctx := context.Background() - t.Run("the live class first, at most four sends at once", func(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - b := newRepublishBed(t) - var old []ReservedVersion - for _, m := range []string{"o1", "o2", "o3", "o4", "o5"} { - old = append(old, b.reserve(t, m)) - } - b.Started = b.now - b.reserve(t, "live") - for i, v := range old { - b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second)) - } - b.now = b.now.Add(time.Hour) // the live one is queued last, after every old one - b.holders = map[string][]string{SeatArtifacts: {"oci"}} - release := make(chan struct{}) - var arrived []string - var inflight, most int - send := b.Send - b.Send = func(ctx context.Context, p Publish) error { - b.mu.Lock() - arrived = append(arrived, p.Name) - inflight++ - most = max(most, inflight) - b.mu.Unlock() - <-release - b.mu.Lock() - inflight-- - b.mu.Unlock() - return send(ctx, p) - } - b.Go = func(f func()) { go f() } - if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 { - t.Fatalf("queued %v (%v), want the live version", q, err) - } - synctest.Wait() // every send the bound lets run waits on release - b.mu.Lock() - first := slices.Clone(arrived) - b.mu.Unlock() - if len(first) != PublishSendsAtOnce || !slices.Contains(first, "live/prog") { - t.Errorf("the sends running at once are %v: want %d, the live one among them", first, PublishSendsAtOnce) - } - close(release) - synctest.Wait() - if len(b.sent) != 6 || most != PublishSendsAtOnce { - t.Errorf("sent %d, at most %d at once: want all six, %d at once", len(b.sent), most, PublishSendsAtOnce) - } - }) - }) - t.Run("a pass lists the keys once for all its sends", func(t *testing.T) { + t.Run("a build that ended before this run is put as again, one after it as live", func(t *testing.T) { b := newRepublishBed(t) - var vs []ReservedVersion - for i := range 6 { - vs = append(vs, b.reserve(t, fmt.Sprintf("m%d", i))) - } - for i, v := range vs { - b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second)) - } - b.Drain(ctx, "oci") - if n := b.queue.listings.Load(); len(b.sent) != 6 || n != 2 { - t.Errorf("sent %d with %d listings, want six sends from one pass and one listing that finds it spent", len(b.sent), n) - } - }) - t.Run("a start makes a live key of a build before it a start key", func(t *testing.T) { - b := newRepublishBed(t) - v := b.reserve(t, "m") + old := b.reserve(t, "o") b.Started = b.now - b.queueAt(t, v, PublishClassLive, b.now) - b.Go = func(func()) {} // the start's drain is not run here - if err := b.Resume(ctx); err != nil { - t.Fatal(err) - } - raw, _, _, _ := b.queue.Get(ctx, QueueKey("oci", "m", v.Version)) - var q QueuedPublish - if json.Unmarshal(raw, &q) != nil || q.Class != PublishClassStart { - t.Errorf("the key after the start is %s, want the class %s: its build did not end in this run", raw, PublishClassStart) + live := b.reserve(t, "l") + b.held = true + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + _, _ = b.Republish(ctx, "o", old.Version) + _, _ = b.Republish(ctx, "l", live.Version) + if len(b.puts) != 2 || b.puts[0].Class != PublishClassStart || b.puts[1].Class != PublishClassLive || + PublishEvent("oci", PublishClassLive, "l", live.Version) != "publish.oci.live.l.0=2E1=2E0" { + t.Errorf("puts %+v, want o again and l live, on publish....", b.puts) } }) - for _, at := range []string{"enqueue", "pass", "start"} { + for _, at := range []string{"put", "send"} { t.Run("a withdrawn build never publishes, at the "+at, func(t *testing.T) { b := newRepublishBed(t) v := b.reserve(t, "m") @@ -361,105 +318,34 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) { if _, ok, err := b.Reserved.Withdraw(ctx, "m", v.Version); err != nil || !ok { t.Fatalf("withdrawn %v (%v)", ok, err) } - switch at { - case "enqueue": - b.queueAt(t, v, PublishClassLive, b.now) - b.Go = func(func()) {} // a key queued before the withdrawal is dropped by the enqueue itself + if at == "put" { if q, _ := b.Republish(ctx, "m", v.Version); len(q) != 0 { t.Errorf("queued %v", q) } - case "pass": - b.queueAt(t, v, PublishClassLive, b.now) - b.Drain(ctx, "oci") - case "start": - b.queueAt(t, v, PublishClassLive, b.now) - b.Go = func(func()) {} - if err := b.Resume(ctx); err != nil { - t.Fatal(err) - } + } else if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" { + t.Errorf("the send of a withdrawn build answered %+v, want it ended", f) } - keys, _ := b.queue.KeysOf(ctx, QueueFilter) - if len(b.sent) != 0 || len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { - t.Errorf("sent %v, keys %v, items ended %v: want nothing sent, the key dropped and m %s's item ended", b.sent, keys, b.ended, v.Version) + if len(b.sent) != 0 || len(b.puts) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { + t.Errorf("sent %v, puts %v, items ended %v: want nothing sent or put and m %s's item ended", b.sent, b.puts, b.ended, v.Version) } }) } - t.Run("a live publish queued mid-pass goes after the sends in flight, before the rest of the start list", func(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - b := newRepublishBed(t) - var old []ReservedVersion - for _, m := range []string{"o1", "o2", "o3", "o4", "o5", "o6"} { - old = append(old, b.reserve(t, m)) - } - b.Started = b.now - b.reserve(t, "live") - for i, v := range old { - b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second)) - } - b.now = b.now.Add(time.Hour) - b.holders = map[string][]string{SeatArtifacts: {"oci"}} - first, then := make(chan struct{}), make(chan struct{}) // the four in flight wait on first, every later send on then - var arrived []string - send := b.Send - b.Send = func(ctx context.Context, p Publish) error { - b.mu.Lock() - arrived = append(arrived, p.Name) - gate := map[bool]chan struct{}{true: first, false: then}[len(arrived) <= PublishSendsAtOnce] - b.mu.Unlock() - <-gate - return send(ctx, p) - } - b.Go = func(f func()) { go f() } - b.drainLater(ctx, "oci") - synctest.Wait() // four start sends in flight, the pass holding o5 and o6 - if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 { - t.Fatalf("queued %v (%v)", q, err) - } - close(first) - synctest.Wait() // the sends after the four: with the live one among them, or o5 and o6 alone when the pass goes on - b.mu.Lock() - next := slices.Clone(arrived[PublishSendsAtOnce:]) - b.mu.Unlock() - if !slices.Contains(next, "live/prog") { - t.Errorf("after the four in flight the sends are %v, want the live one among them, the pass listed again", next) - } - close(then) - synctest.Wait() - if len(arrived) != 7 { - t.Errorf("sent %v, want all seven", arrived) - } - }) - }) - t.Run("a build a newer patch or release of its series supersedes is retired at the pass and at the start, never by a newer series", func(t *testing.T) { + t.Run("a build a newer patch or release of its series supersedes is ended at the send, never by a newer series", func(t *testing.T) { b := newRepublishBed(t) v := b.reserve(t, "m") - b.holders = map[string][]string{SeatArtifacts: {"oci"}} if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil { t.Fatal(err) } - b.queueAt(t, v, PublishClassStart, b.now) - b.Drain(ctx, "oci") - if len(b.sent) != 1 || len(b.ended) != 0 { - t.Errorf("with a standing 0.2: sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", b.sent, b.ended, v.Version) + if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); !f.Sent || len(b.sent) != 1 || len(b.ended) != 0 { + t.Errorf("with a standing 0.2: %+v, sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", f, b.sent, b.ended, v.Version) } rel := &reserveMem{mem: newMem()} raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.3"}, {Fingerprint: "g", Version: "0.1.9"}}}) _, _ = rel.Create(ctx, ReleasedKey("m"), raw) b.Reserved.Releases = rel b.sent = nil - b.queueAt(t, v, PublishClassStart, b.now) - b.Drain(ctx, "oci") - if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { - t.Errorf("at the pass: sent %v, ended %v, want m %s retired by the release 0.1.9", b.sent, b.ended, v.Version) - } - b.ended = nil - b.queueAt(t, v, PublishClassStart, b.now) - b.Go = func(func()) {} - if err := b.Resume(ctx); err != nil { - t.Fatal(err) - } - if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { - t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version) + if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { + t.Errorf("at the send: %+v, sent %v, ended %v, want m %s retired by the release 0.1.9", f, b.sent, b.ended, v.Version) } only := &reserveMem{mem: newMem()} raw, _ = json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.0"}}}) @@ -473,13 +359,27 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) { t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) { b := newRepublishBed(t) v := b.reserve(t, "m") - b.queueAt(t, v, PublishClassStart, b.now) if _, err := b.Reserved.Reserve(ctx, "m", "0.1", []string{"m:prog"}, Source{Fingerprint: "src2:other"}); err != nil { t.Fatal(err) } - b.Drain(ctx, "oci") - if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { - t.Errorf("sent %v, ended %v: want nothing sent and the item ended", b.sent, b.ended) + if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { + t.Errorf("%+v, sent %v, ended %v: want nothing sent and the item ended", f, b.sent, b.ended) + } + }) + t.Run("an item whose artifacts are published or refused as not stored ends unsent", func(t *testing.T) { + b := newRepublishBed(t) + v := b.reserve(t, "m") + b.refuse = "not stored: no blob" + if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 { + t.Errorf("a not stored refusal answered %+v, sent %d: want it ended after one send", f, len(b.sent)) + } + if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 { + t.Errorf("again: %+v, sent %d: want it ended unsent", f, len(b.sent)) + } + b.refuse = "the holder is full" + w := b.reserve(t, "n") + if f := b.SendQueued(ctx, b.queueAt(w, PublishClassLive, b.now)); f.Failed == "" || f.Sent { + t.Errorf("a refusal in other words answered %+v, want it failed, handed back", f) } }) } @@ -527,7 +427,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) { t.Errorf("a state one page holds is answered capped: %+v", a.Read) } queue.All = func() (StatePage, error) { return StatePage{Values: vals, More: true, Keys: 5000, Bytes: 3 << 20}, nil } - if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishQueueState || + if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishesLocal || c.Read[0].KeysRead != 2000 || c.Read[0].StateKeys != 5000 || c.Read[0].StateBytes != 3<<20 || c.Total != 2000 { t.Errorf("a state past its one page answered capped %v, read %+v (%v): want capped, 2000 keys read of 5000, its size said (H-A-293)", c.Capped, c.Read, err) }