B4: the publish items travel on the work queue publishes, live first
The bucket publish-queue listed and reread itself before each send; its items now ride the builder's work queue publishes, pulled per holder and class by consumers the builder makes as it learns a holder, the live class first, a failed send handed back with the pause, an item acknowledged when its publish is heard and terminated when its build is withdrawn, superseded or dropped. builder_publish_drop and builder_publish_first act through reasks after the hand-act log; builder_publish_queue reads the queue through the runtime. The old state's items are moved once at the start, then dropped through the bus module. Builds: design 47 H2 §793, §820; H-A-290, H-A-291, H-A-440, H-A-446 (2), H-A-448
This commit is contained in:
@@ -433,8 +433,9 @@ func main() {
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
|
||||
Newest: newestVia(stdio.Ask), Steps: r}))
|
||||
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState)
|
||||
tools = append(tools, republishTools(readerOf(qs), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
as := stdio.State(versions.ActedState)
|
||||
tools = append(tools, publishActTools(buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
tools = append(tools, republishTools(workQueueReader(stdio.BusState, versions.PublishesLocal), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
ws := stdio.State(build.WaitsState)
|
||||
wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) {
|
||||
e, err := ws.Get(key)
|
||||
|
||||
@@ -4,11 +4,13 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
@@ -53,6 +55,95 @@ func onePage(page func(after string, maxBytes int, budget time.Duration) (stdio.
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// workQueuePageItems and workQueueItemsRead bound the read of the work queue publishes: pages of 500 items, at most 4,000 read
|
||||
// (H-A-293: past them the answer is capped, naming the items read and the queue's size).
|
||||
const (
|
||||
workQueuePageItems = 500
|
||||
workQueueItemsRead = 4000
|
||||
)
|
||||
|
||||
// workQueueReader is the builder's work queue local as the versions package's reader (design 47 H2 §793, H-A-448): its items read
|
||||
// through the tool runtime's stream read in pages, each keyed by its sequence, and its size through the work queue's size; a
|
||||
// read past workQueueItemsRead is capped. call is stdio.BusState.
|
||||
func workQueueReader(call func(method string, params any) (json.RawMessage, error), local string) versions.Reader {
|
||||
none := func() ([]string, error) { return nil, errors.ErrUnsupported }
|
||||
return versions.Reader{Keys: none, All: func() (versions.StatePage, error) {
|
||||
out := versions.StatePage{Values: map[string]json.RawMessage{}}
|
||||
from := uint64(0)
|
||||
for {
|
||||
raw, err := call(buscalls.StreamRead, buscalls.StreamReadRequest{Stream: local, From: from, Limit: workQueuePageItems})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var pg buscalls.StreamPage
|
||||
if err := json.Unmarshal(raw, &pg); err != nil {
|
||||
return out, fmt.Errorf("the work queue %s answered what is no page of items: %w", local, err)
|
||||
}
|
||||
for _, m := range pg.Messages {
|
||||
out.Values[strconv.FormatUint(m.Sequence, 10)] = m.Body
|
||||
}
|
||||
if pg.Next == 0 || len(pg.Messages) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
if len(out.Values) >= workQueueItemsRead {
|
||||
out.More = true
|
||||
break
|
||||
}
|
||||
from = pg.Next
|
||||
}
|
||||
raw, err := call(buscalls.WorkQueueSize, buscalls.WorkQueueRequest{Queue: local})
|
||||
var size buscalls.WorkQueueSized
|
||||
if err == nil {
|
||||
err = json.Unmarshal(raw, &size)
|
||||
}
|
||||
if err != nil {
|
||||
out.InfoWords = fmt.Sprintf("the work queue's size cannot be read: %v", err)
|
||||
} else {
|
||||
out.Keys, out.Bytes = size.Count, size.Bytes
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// publishActTools are builder_publish_drop and builder_publish_first (design 47 H2 §793): hand acts, recorded in the hand-act log
|
||||
// first, then written to reasks for the builder, which owns the work queue, to perform and answer.
|
||||
func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
tool := func(name, mode, desc, what string) stdio.Tool {
|
||||
return stdio.Tool{Name: name, Description: desc,
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "version", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the item's module"},
|
||||
"version": map[string]any{"type": "string", "description": "the item's version, MAJOR.MINOR.PATCH"},
|
||||
"why": map[string]any{"type": "string", "description": "why, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewPublishAct(mode, str(args, "module"), str(args, "version"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("%s %s %s by hand", what, str(args, "module"), str(args, "version")),
|
||||
Why: str(args, "why"), Cause: name, Verb: mode}); err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was done: %w", err)
|
||||
}
|
||||
raw, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Refused {
|
||||
return nil, errors.New(a.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "answer": a.Answer}, nil
|
||||
}}
|
||||
}
|
||||
return []stdio.Tool{
|
||||
tool(versions.PublishDropTool, build.ModePublishDrop, versions.PublishDropDescription, "dropped the publish of"),
|
||||
tool(versions.PublishFirstTool, build.ModePublishFirst, versions.PublishFirstDescription, "put first the publish of"),
|
||||
}
|
||||
}
|
||||
|
||||
// republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act:
|
||||
// it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer
|
||||
// with the versions it queued.
|
||||
|
||||
@@ -3,10 +3,15 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
@@ -60,4 +65,58 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
t.Errorf("health four minutes in = %+v, want healthy", a)
|
||||
}
|
||||
})
|
||||
// Design 47 H2 §793, H-A-448: builder_publish_queue reads the work queue publishes through the runtime's stream read, in pages,
|
||||
// and a read past its bound is capped with the queue's size from the work queue's size; builder_publish_drop and
|
||||
// builder_publish_first record the hand act first, then write their reasks value.
|
||||
t.Run("the work queue is read in pages, capped past its bound", func(t *testing.T) {
|
||||
for _, total := range []int{1200, 9000} {
|
||||
var methods []string
|
||||
call := func(method string, params any) (json.RawMessage, error) {
|
||||
methods = append(methods, method)
|
||||
if method == buscalls.WorkQueueSize {
|
||||
return json.Marshal(buscalls.WorkQueueSized{Fill: buscalls.Fill{Count: uint64(total), Bytes: 4096}})
|
||||
}
|
||||
req := params.(buscalls.StreamReadRequest)
|
||||
var pg buscalls.StreamPage
|
||||
for n := req.From + 1; n <= uint64(total) && len(pg.Messages) < req.Limit; n++ {
|
||||
pg.Messages = append(pg.Messages, buscalls.StreamMessage{Sequence: n, Body: json.RawMessage(`{"holder":"oci","class":"again"}`)})
|
||||
pg.Next = n
|
||||
}
|
||||
if pg.Next == uint64(total) {
|
||||
pg.Next = 0
|
||||
}
|
||||
return json.Marshal(pg)
|
||||
}
|
||||
pg, err := workQueueReader(call, versions.PublishesLocal).All()
|
||||
capped := total > workQueueItemsRead
|
||||
if err != nil || pg.More != capped || len(pg.Values) != min(total, workQueueItemsRead) || (capped && pg.Keys != uint64(total)) {
|
||||
t.Errorf("%d items: read %d, more %v, keys %d (%v) after %v", total, len(pg.Values), pg.More, pg.Keys, err, methods)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("drop and first record the hand act, then ask the builder", func(t *testing.T) {
|
||||
var order []string
|
||||
ask := func(key string, value []byte) (json.RawMessage, error) {
|
||||
order = append(order, "ask "+key)
|
||||
r, _ := build.DecodeReAsk(value)
|
||||
r.Answer = "done"
|
||||
return json.Marshal(r)
|
||||
}
|
||||
acts := meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
order = append(order, "record")
|
||||
return json.RawMessage(`{"id":"h1"}`), nil
|
||||
}}
|
||||
tools := publishActTools(ask, func() time.Time { return time.Unix(5, 0) }, acts)
|
||||
for _, tool := range tools {
|
||||
if _, err := tool.Run(map[string]any{"module": "m", "version": "0.1.2", "why": "a test"}); err != nil {
|
||||
t.Fatalf("%s: %v", tool.Name, err)
|
||||
}
|
||||
}
|
||||
if len(tools) != 2 || fmt.Sprint(order) != "[record ask publish-drop.5000000000 record ask publish-first.5000000000]" {
|
||||
t.Errorf("tools %d, order %v: want each hand act recorded before its reasks value", len(tools), order)
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"module": "m", "why": "no version"}); err == nil {
|
||||
t.Error("a drop naming no version is not refused")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
@@ -39,6 +40,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/operatorask"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
@@ -268,10 +270,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.
|
||||
acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider}
|
||||
// A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G
|
||||
// §588): its states publish-queue and acted, bound once the bus holds them.
|
||||
queueKV := process.AwaitBucket(ctx, js, conn, own.queue, "no version is published again on an event or by builder_publish_again", logger.Printf)
|
||||
// §588): its items on the work queue publishes its requires asks for, named by the bus's binding (design 47 H2 §793, H-A-448).
|
||||
actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf)
|
||||
republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}},
|
||||
publishesBinding := process.NewBinding(meshbroker.Seat, cred.Module, calls.CallKept, logger.Printf, versions.PublishesLocal)
|
||||
if err := publishesBinding.Register(hub); err != nil {
|
||||
return fmt.Errorf("hearing the bus's binding of %s: %w", versions.PublishesLocal, err)
|
||||
}
|
||||
pq := &publishQueue{js: js, logf: logger.Printf,
|
||||
binding: func() (membership.BusBinding, error) { return publishesBinding.Names(versions.PublishesLocal) }}
|
||||
republish := &versions.Republisher{Reserved: reserved.r, Put: pq.put,
|
||||
Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk),
|
||||
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
|
||||
Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
|
||||
@@ -300,6 +307,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -318,15 +326,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
},
|
||||
markHeld: reserved.markHeld, moveElsewhere: reserved.movedElsewhere,
|
||||
allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { return reserved.r.All(ctx) },
|
||||
send: republish.SendQueued,
|
||||
retired: reserved.r.Retired} // only each module's newest done build is asked at the start (H-A-291 (1))
|
||||
// A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)).
|
||||
republish.Ended = items.Done
|
||||
republish.Ended = items.Ended
|
||||
pq.take = items.take
|
||||
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
|
||||
wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept)
|
||||
acts.Republish = republish.Republish
|
||||
acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) {
|
||||
return pq.drop(ctx, items, module, version, by, why)
|
||||
}
|
||||
acts.PublishFirst = func(ctx context.Context, module, version, _, _ string) (string, error) {
|
||||
return pq.first(ctx, items, module, version)
|
||||
}
|
||||
// builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does.
|
||||
acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf)
|
||||
again := &republishing{rp: republish}
|
||||
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
|
||||
decided := &decisions{subject: oc.SubjectDecided(cred.Module)}
|
||||
retakeActed := awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}
|
||||
@@ -340,8 +355,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return fmt.Errorf("hearing the operator's answers: %w", err)
|
||||
}
|
||||
retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}}
|
||||
queueKV.Ready(again.bound(ctx, js, logger.Printf))
|
||||
actedKV.Ready(again.bound(ctx, js, logger.Printf))
|
||||
// Once the bus names the work queue: what waits on it is pulled, and the old state's items are put on it once (§793).
|
||||
publishesBinding.Start()
|
||||
go func() {
|
||||
publishesBinding.Ask(versions.PublishesLocal)
|
||||
if err := pq.learnAll(ctx); err != nil {
|
||||
logger.Printf("the publishes waiting on the work queue %s are not pulled yet: %v; each holder's are pulled when an item of it is put", versions.PublishesLocal, err)
|
||||
}
|
||||
moveOld(ctx, js, own.queue, cred.Module, republish, calls.Call, logger.Printf)
|
||||
}()
|
||||
errs := make(chan error, 10)
|
||||
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466)
|
||||
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
||||
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder.
|
||||
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
|
||||
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and asks the
|
||||
// bus module to drop the old state.
|
||||
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
b := startBed(t)
|
||||
ctx := b.ctx
|
||||
bind := membership.BusBinding{Local: versions.PublishesLocal, Stream: "mesh-delivery-builder_publishes", Subject: "mesh.bus.mesh-delivery-builder.publishes"}
|
||||
if _, err := b.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: bind.Stream, Subjects: []string{bind.Subject + ".>"},
|
||||
Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
var mu sync.Mutex
|
||||
var sent []string
|
||||
published := map[string]bool{} // the modules whose publish is heard
|
||||
sentCh := make(chan string, 200)
|
||||
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5},
|
||||
now: time.Now, logf: t.Logf,
|
||||
reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil
|
||||
},
|
||||
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
|
||||
mu.Lock()
|
||||
sent = append(sent, q.Module)
|
||||
mu.Unlock()
|
||||
sentCh <- q.Module
|
||||
return versions.SendFate{Sent: true}
|
||||
}}
|
||||
pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: func() (membership.BusBinding, error) { return bind, nil }}
|
||||
item := func(module, class string) versions.QueuedPublish {
|
||||
return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()}
|
||||
}
|
||||
away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for i := range 100 {
|
||||
away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart))
|
||||
}
|
||||
away(item("dropme", versions.PublishClassStart))
|
||||
away(item("live", versions.PublishClassLive))
|
||||
if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil {
|
||||
t.Fatalf("drop: %v", err)
|
||||
} else {
|
||||
t.Log(answer)
|
||||
}
|
||||
if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup {
|
||||
t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err)
|
||||
}
|
||||
if err := pq.learnAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case first := <-sentCh:
|
||||
if first != "live" {
|
||||
t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("nothing was sent")
|
||||
}
|
||||
for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the again items were not sent")
|
||||
}
|
||||
}
|
||||
deadline := time.After(60 * time.Second)
|
||||
for { // each publish sent is heard: its item acknowledged, the next sent
|
||||
mu.Lock()
|
||||
n, done := len(sent), slices.Clone(sent)
|
||||
for _, m := range done {
|
||||
published[m] = true
|
||||
}
|
||||
mu.Unlock()
|
||||
if n >= 101 {
|
||||
break
|
||||
}
|
||||
for _, m := range done {
|
||||
items.heard(ctx, m, "0.1.0")
|
||||
}
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-deadline:
|
||||
t.Fatalf("sent %d of 101", n)
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
if slices.Contains(sent, "dropme") || len(sent) != 101 {
|
||||
t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme"))
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
// The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked).
|
||||
old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
call := func(string, any) (json.RawMessage, error) {
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
||||
}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
for _, m := range []string{"m1", "m2", "m3"} {
|
||||
v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"})
|
||||
if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) {
|
||||
data, _ := json.Marshal(q)
|
||||
ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return ack.Duplicate, nil
|
||||
}, Started: time.Now(), Logf: t.Logf}
|
||||
var drops []string
|
||||
drop := func(tool string, body any) (json.RawMessage, error) {
|
||||
drops = append(drops, fmt.Sprint(tool, body))
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
for range 2 {
|
||||
moveOld(ctx, b.js, builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState), "mesh-delivery-builder", rp, drop, t.Logf)
|
||||
}
|
||||
st, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 || len(drops) != 2 {
|
||||
t.Errorf("the move run twice left %v on the work queue, dropped %v: want each of the three once, as again, and the old state dropped through the bus module", info.State.Subjects, drops)
|
||||
}
|
||||
}
|
||||
@@ -7,20 +7,25 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588): each reservation of a
|
||||
// done build is a publish item in the builder's re-ask state, the mechanism builder_reserve_done and publish again use. It is
|
||||
// asked when the build ends and at the builder's start; each try queues the publish and writes the item again with one more try
|
||||
// and a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue.
|
||||
// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588; H2 §793): each
|
||||
// reservation of a done build is a publish item in the builder's re-ask state, asked when the build ends and at the builder's
|
||||
// start; its try puts the publish on the work queue publishes, and each send counts one try in the item and hands a failed one
|
||||
// back with a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue.
|
||||
|
||||
// publishSettings are the settings publish-pause, publish-pause-max and publish-tries.
|
||||
type publishSettings struct {
|
||||
@@ -28,8 +33,8 @@ type publishSettings struct {
|
||||
Tries int
|
||||
}
|
||||
|
||||
// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 8 tries.
|
||||
var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 8}
|
||||
// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 5 tries (H-A-446 (2)).
|
||||
var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 5}
|
||||
|
||||
// pauseAfter is the pause after the try-th try: the setting doubled per try, up to the maximum.
|
||||
func (s publishSettings) pauseAfter(try int) time.Duration {
|
||||
@@ -61,10 +66,15 @@ type publishItems struct {
|
||||
// publish there (H-A-275). Nil does neither.
|
||||
markHeld func(ctx context.Context, v versions.ReservedVersion, held []versions.HeldArtifact)
|
||||
moveElsewhere func(ctx context.Context, v versions.ReservedVersion, j versions.Judged) error
|
||||
set publishSettings
|
||||
now func() time.Time
|
||||
after func(time.Duration) <-chan time.Time
|
||||
logf func(string, ...any)
|
||||
// send sends one item pulled from the work queue (Republisher.SendQueued); nil sends none.
|
||||
send func(ctx context.Context, q versions.QueuedPublish) versions.SendFate
|
||||
set publishSettings
|
||||
now func() time.Time
|
||||
after func(time.Duration) <-chan time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
held map[string][]heldItem // "<module> <version>": the items sent, their publish not yet heard
|
||||
}
|
||||
|
||||
func (p *publishItems) wait(ctx context.Context, d time.Duration) bool {
|
||||
@@ -130,8 +140,11 @@ func (p *publishItems) AskAllDone(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Done deletes a version's publish item: its release is written.
|
||||
// Done deletes a version's publish item and acknowledges its held items: its release is written.
|
||||
func (p *publishItems) Done(ctx context.Context, module, version string) {
|
||||
for _, h := range p.settle(module, version, func(heldItem) bool { return true }) {
|
||||
_ = h.m.Ack()
|
||||
}
|
||||
kv := p.reasks()
|
||||
if kv == nil {
|
||||
return
|
||||
@@ -141,9 +154,10 @@ func (p *publishItems) Done(ctx context.Context, module, version string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Try waits for the item's next try, then, while the version is still reserved, queues its publish and writes the item again with
|
||||
// one more try and the pause after it; once the tries are spent it ends the item loudly. A version no longer reserved (released,
|
||||
// dropped or taken over) has its item deleted; one whose artifacts are published keeps it until its release is written.
|
||||
// Try waits for the item's next try, then, while the version is still reserved, puts its publish on the work queue publishes
|
||||
// under its message id (design 47 H2 §793; H-A-448): an item the queue holds already is put once. It counts no try: each send
|
||||
// does, and hands its item back with the pause (take). A put that queued nothing is put again after the pause. A version no
|
||||
// longer reserved (released, dropped or taken over) has its item deleted.
|
||||
func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk) {
|
||||
if !p.wait(ctx, r.NextTry.Sub(p.now())) {
|
||||
return
|
||||
@@ -157,37 +171,215 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre
|
||||
p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err)
|
||||
return
|
||||
}
|
||||
if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn: never because its artifacts are published (A-456b, H-A-290 (2))
|
||||
if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn (A-456b, H-A-290 (2))
|
||||
p.logf("%s %s: the publish item is ended: %s", r.Module, r.Version, why)
|
||||
_ = kv.Delete(ctx, e.Key())
|
||||
return
|
||||
}
|
||||
queued, err := p.republish(ctx, r.Module, r.Version)
|
||||
if err == nil && len(queued) > 0 {
|
||||
return
|
||||
}
|
||||
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
|
||||
p.logf("%s %s: the publish is not on the work queue %s (%v); it is put again after %s", r.Module, r.Version, versions.PublishesLocal, err, p.set.pauseAfter(r.Tries))
|
||||
if data, err := json.Marshal(r); err == nil {
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
p.logf("%s: the publish item could not be written: %v", e.Key(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// itemMsg is what take reads of an item pulled from the work queue (jetstream.Msg).
|
||||
type itemMsg interface {
|
||||
Data() []byte
|
||||
Ack() error
|
||||
NakWithDelay(time.Duration) error
|
||||
Term() error
|
||||
}
|
||||
|
||||
// take handles one item pulled from the work queue (§793; H-A-440 (2), H-A-448): an item whose publish item ended (dropped, or
|
||||
// its tries spent) or whose version never publishes is terminated; one whose artifacts are all published is acknowledged; once
|
||||
// publish-tries sends are spent it is terminated and version-unreleased said; while a published of it may be unheard it is handed
|
||||
// back with the pause, and one the registries' record confirms is acknowledged (H-A-274); else it is sent, the try counted in its
|
||||
// publish item, and a send that fails handed back with the pause as its delay, so the queue delivers it again and nothing is put
|
||||
// again. A sent item is acknowledged when its publish is heard (heard, Done) and delivered again after the ack wait, unheard.
|
||||
func (p *publishItems) take(ctx context.Context, m itemMsg) {
|
||||
var q versions.QueuedPublish
|
||||
if err := json.Unmarshal(m.Data(), &q); err != nil || q.Module == "" || q.Version == "" {
|
||||
p.logf("an item of the work queue %s that is no publish is terminated: %v", versions.PublishesLocal, err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
kv := p.reasks()
|
||||
if kv == nil {
|
||||
p.logf("%s %s: the re-ask state is not on the bus yet; the item is handed back", q.Module, q.Version)
|
||||
_ = m.NakWithDelay(p.set.Pause)
|
||||
return
|
||||
}
|
||||
key := versions.PublishItemKey(q.Module, q.Version)
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) { // put by builder_publish_again or the move: its tries are counted from now
|
||||
if err = p.Ask(ctx, q.Module, q.Version); err == nil {
|
||||
e, err = kv.Get(ctx, key)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("%s %s: its publish item cannot be read (%v); the item is handed back", q.Module, q.Version, err)
|
||||
_ = m.NakWithDelay(p.set.Pause)
|
||||
return
|
||||
}
|
||||
r, err := build.DecodeReAsk(e.Value())
|
||||
if err != nil {
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if r.Answer != "" {
|
||||
p.logf("%s %s at %s is terminated: its publish item ended: %s", q.Module, q.Version, q.Holder, r.Answer)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
v, found, err := p.reserved(ctx, q.Module, q.Version)
|
||||
if err != nil {
|
||||
p.logf("%s %s: its reservation cannot be read (%v); the item is handed back", q.Module, q.Version, err)
|
||||
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
|
||||
return
|
||||
}
|
||||
if why := versions.NotStanding(v, found); why != "" {
|
||||
p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, why)
|
||||
_ = m.Term()
|
||||
_ = kv.Delete(ctx, key)
|
||||
return
|
||||
}
|
||||
if allPublished(v, q) {
|
||||
p.logf("%s %s at %s is acknowledged: its artifacts are published", q.Module, q.Version, q.Holder)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if r.Tries >= p.set.Tries {
|
||||
_ = m.Term()
|
||||
r.Answer = fmt.Sprintf("the publish of %s %s was tried %d times and the version is not released; builder_publish_again tries it again", r.Module, r.Version, r.Tries)
|
||||
r.Answered = p.now().UTC()
|
||||
p.logf("%s: %s", e.Key(), r.Answer)
|
||||
p.logf("%s: %s", key, r.Answer)
|
||||
if p.endedWith != nil {
|
||||
p.endedWith(ctx, v, "the publish ended after "+fmt.Sprint(r.Tries)+" tries")
|
||||
}
|
||||
} else if why, gone := p.notAgain(ctx, v); gone {
|
||||
return // moved to another number, whose own item is asked (H-A-275)
|
||||
} else if why != "" {
|
||||
// A published of the version may be unheard, or the registry's record ended it: no ask, and no try spent (H-A-272, H-A-274).
|
||||
p.logf("%s %s: the publish is not asked again: %s; tried again after %s", r.Module, r.Version, why, p.set.pauseAfter(r.Tries))
|
||||
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
|
||||
} else {
|
||||
r.Tries++
|
||||
if _, err := p.republish(ctx, r.Module, r.Version); err != nil {
|
||||
p.logf("%s %s: try %d queued no publish: %v", r.Module, r.Version, r.Tries, err)
|
||||
}
|
||||
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
|
||||
p.write(ctx, kv, key, r, e.Revision())
|
||||
return
|
||||
}
|
||||
if why, gone := p.notAgain(ctx, v); gone {
|
||||
_ = m.Term() // moved to another number, whose own item is asked (H-A-275)
|
||||
return
|
||||
} else if why != "" {
|
||||
if v2, ok, err := p.reserved(ctx, q.Module, q.Version); err == nil && ok && allPublished(v2, q) {
|
||||
p.logf("%s %s at %s is acknowledged: %s", q.Module, q.Version, q.Holder, why)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
p.logf("%s %s: the publish is not sent again: %s; handed back for %s", q.Module, q.Version, why, p.set.pauseAfter(r.Tries))
|
||||
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
|
||||
return
|
||||
}
|
||||
fate := versions.SendFate{Failed: "this builder sends no publish"}
|
||||
if p.send != nil {
|
||||
fate = p.send(ctx, q)
|
||||
}
|
||||
if fate.Ended != "" {
|
||||
p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, fate.Ended)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
r.Tries++
|
||||
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
|
||||
p.write(ctx, kv, key, r, e.Revision())
|
||||
if fate.Failed != "" {
|
||||
p.logf("%s %s at %s: try %d sent no publish: %s; handed back for %s", q.Module, q.Version, q.Holder, r.Tries, fate.Failed, p.set.pauseAfter(r.Tries))
|
||||
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
|
||||
return
|
||||
}
|
||||
p.hold(q, m)
|
||||
}
|
||||
|
||||
// write writes a publish item at its revision; a newer write wins.
|
||||
func (p *publishItems) write(ctx context.Context, kv jetstream.KeyValue, key string, r build.ReAsk, rev uint64) {
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
p.logf("%s: the publish item could not be written: %v", e.Key(), err)
|
||||
if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
p.logf("%s: the publish item could not be written: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// allPublished says none of q's artifacts is missing from v: its publish is heard.
|
||||
func allPublished(v versions.ReservedVersion, q versions.QueuedPublish) bool {
|
||||
missing := map[string]bool{}
|
||||
for _, ref := range v.Missing() {
|
||||
missing[ref] = true
|
||||
}
|
||||
for ref := range q.Artifacts {
|
||||
if missing[ref] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// heldItem is an item sent and not yet acknowledged: its publish is not heard yet.
|
||||
type heldItem struct {
|
||||
q versions.QueuedPublish
|
||||
m itemMsg
|
||||
}
|
||||
|
||||
// hold keeps a sent item until its publish is heard.
|
||||
func (p *publishItems) hold(q versions.QueuedPublish, m itemMsg) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.held == nil {
|
||||
p.held = map[string][]heldItem{}
|
||||
}
|
||||
k := q.Module + " " + q.Version
|
||||
p.held[k] = append(p.held[k], heldItem{q: q, m: m})
|
||||
}
|
||||
|
||||
// settle takes every held item of module's version that pick picks out of the held ones.
|
||||
func (p *publishItems) settle(module, version string, pick func(heldItem) bool) []heldItem {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
k := module + " " + version
|
||||
var out, keep []heldItem
|
||||
for _, h := range p.held[k] {
|
||||
if pick(h) {
|
||||
out = append(out, h)
|
||||
} else {
|
||||
keep = append(keep, h)
|
||||
}
|
||||
}
|
||||
if len(keep) == 0 {
|
||||
delete(p.held, k)
|
||||
} else {
|
||||
p.held[k] = keep
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heard acknowledges each held item of module's version whose artifacts are all published now: its publish is heard (§793).
|
||||
func (p *publishItems) heard(ctx context.Context, module, version string) {
|
||||
v, found, err := p.reserved(ctx, module, version)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
for _, h := range p.settle(module, version, func(h heldItem) bool { return allPublished(v, h.q) }) {
|
||||
p.logf("%s %s at %s is acknowledged: its publish is heard", module, version, h.q.Holder)
|
||||
_ = h.m.Ack()
|
||||
}
|
||||
}
|
||||
|
||||
// Ended deletes a version's publish item and terminates its held items: its build is withdrawn or superseded (H-A-290 (2),
|
||||
// H-A-291 (1)); an item still on the work queue is terminated when it is pulled.
|
||||
func (p *publishItems) Ended(ctx context.Context, module, version string) {
|
||||
p.Done(ctx, module, version)
|
||||
for _, h := range p.settle(module, version, func(heldItem) bool { return true }) {
|
||||
_ = h.m.Term()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -312,10 +504,11 @@ func publishSettingsOf(getenv func(string) string) (publishSettings, error) {
|
||||
return publishSettings{Pause: time.Duration(pause) * time.Second, PauseMax: time.Duration(pmax) * time.Minute, Tries: tries}, nil
|
||||
}
|
||||
|
||||
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b): a done build asks its item, a
|
||||
// release, a drop and a takeover end it, and a release is recorded in releases, the last kept of each module.
|
||||
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b, H2 §793): a done build asks its
|
||||
// item, a release ends it and acknowledges its held items, a drop and a takeover end it and terminate them, a published heard
|
||||
// acknowledges those it publishes, and a release is recorded in releases, the last kept of each module.
|
||||
func wirePublishItems(rv *reservations, items *publishItems, releases versions.ReserveStore, kept int) {
|
||||
rv.built, rv.released = items.Ask, items.Done
|
||||
rv.built, rv.released, rv.itemEnded, rv.itemHeard = items.Ask, items.Done, items.Ended, items.heard
|
||||
rv.r.ReleasedKept, rv.r.Releases = kept, releases
|
||||
}
|
||||
|
||||
@@ -330,3 +523,366 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-448): asked for in
|
||||
// its requires and named by the bus's binding; each item on `<binding subject>.publish.<holder>.<class>.<module>.<version>` under
|
||||
// the message id `<holder>.<class>.<module>.<version>`. The builder makes the pull consumers at run time as it learns a holder,
|
||||
// `publishes-<holder>-<class>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live
|
||||
// class first, at most four sends at once per holder.
|
||||
type publishQueue struct {
|
||||
js jetstream.JetStream
|
||||
binding func() (membership.BusBinding, error)
|
||||
ackWait time.Duration
|
||||
take func(ctx context.Context, m itemMsg)
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
pulling map[string]bool
|
||||
}
|
||||
|
||||
// The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's
|
||||
// largest; the builder ends an item at publish-tries anyway.
|
||||
const (
|
||||
publishesMaxAckPending = versions.PublishSendsAtOnce
|
||||
publishesMaxDeliver = 10
|
||||
)
|
||||
|
||||
// publishesConsumer is the durable consumer of holder's items of class.
|
||||
func publishesConsumer(holder, class string) string {
|
||||
return versions.PublishesLocal + "-" + holder + "-" + class
|
||||
}
|
||||
|
||||
// subject is the item's subject on the bus.
|
||||
func (pq *publishQueue) subject(b membership.BusBinding, q versions.QueuedPublish) string {
|
||||
return b.Subject + "." + versions.PublishEvent(q.Holder, q.Class, q.Module, q.Version)
|
||||
}
|
||||
|
||||
// put puts q on the work queue under its message id; duplicate says the queue holds that id within its window. A put the full
|
||||
// queue refuses is versions.ErrQueueFull. The item's holder is learnt: its pull starts.
|
||||
func (pq *publishQueue) put(ctx context.Context, q versions.QueuedPublish) (bool, error) {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
data, err := json.Marshal(q)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ack, err := pq.js.PublishMsg(ctx, &nats.Msg{Subject: pq.subject(b, q), Data: data}, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
|
||||
if err != nil {
|
||||
if w := err.Error(); strings.Contains(w, "maximum bytes") || strings.Contains(w, "maximum messages") || errors.Is(err, jetstream.ErrMaxBytesExceeded) {
|
||||
return false, fmt.Errorf("%w: %v", versions.ErrQueueFull, err)
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
pq.learn(ctx, q.Holder)
|
||||
return ack.Duplicate, nil
|
||||
}
|
||||
|
||||
// learnAll learns every holder whose items the work queue holds: at the start, so what waits there is pulled.
|
||||
func (pq *publishQueue) learnAll(ctx context.Context) error {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
st, err := pq.js.Stream(ctx, b.Stream)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.>"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for subj := range info.State.Subjects {
|
||||
if holder, _, ok := strings.Cut(strings.TrimPrefix(subj, b.Subject+".publish."), "."); ok {
|
||||
pq.learn(ctx, holder) // a holder is a module, its name one token
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// learn starts holder's pull once.
|
||||
func (pq *publishQueue) learn(ctx context.Context, holder string) {
|
||||
pq.mu.Lock()
|
||||
if pq.pulling == nil {
|
||||
pq.pulling = map[string]bool{}
|
||||
}
|
||||
if pq.pulling[holder] {
|
||||
pq.mu.Unlock()
|
||||
return
|
||||
}
|
||||
pq.pulling[holder] = true
|
||||
pq.mu.Unlock()
|
||||
if err := pq.pull(ctx, holder); err != nil {
|
||||
pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err)
|
||||
pq.mu.Lock()
|
||||
delete(pq.pulling, holder)
|
||||
pq.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item
|
||||
// waits while a live one waits on the queue.
|
||||
func (pq *publishQueue) pull(ctx context.Context, holder string) error {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chans := map[string]chan jetstream.Msg{}
|
||||
var live jetstream.Consumer
|
||||
for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { // the again first: the live is first by the dispatch, not by luck
|
||||
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class),
|
||||
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err)
|
||||
}
|
||||
ch := make(chan jetstream.Msg, publishesMaxAckPending)
|
||||
cc, err := c.Consume(func(m jetstream.Msg) { ch <- m }, jetstream.PullMaxMessages(publishesMaxAckPending))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s is not pulled: %w", publishesConsumer(holder, class), err)
|
||||
}
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
chans[class] = ch
|
||||
if class == versions.PublishClassLive {
|
||||
live = c
|
||||
}
|
||||
}
|
||||
liveWaits := func() bool {
|
||||
info, err := live.Info(ctx)
|
||||
return err == nil && info.NumPending > 0
|
||||
}
|
||||
pq.logf("pulling the publishes of %s, the live first", holder)
|
||||
go pq.dispatch(ctx, chans[versions.PublishClassLive], chans[versions.PublishClassStart], liveWaits)
|
||||
return nil
|
||||
}
|
||||
|
||||
// dispatch runs the items of one holder, at most four at once, every live item before an again one: an again item pulled while
|
||||
// a live one waits on the queue (liveWaits) runs after it.
|
||||
func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstream.Msg, liveWaits func() bool) {
|
||||
sem := make(chan struct{}, versions.PublishSendsAtOnce)
|
||||
run := func(m jetstream.Msg) {
|
||||
go func() {
|
||||
defer func() { <-sem }()
|
||||
pq.take(ctx, m)
|
||||
}()
|
||||
}
|
||||
var spare jetstream.Msg // an again item pulled while a live one waited
|
||||
for {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
select {
|
||||
case m := <-live:
|
||||
run(m)
|
||||
continue
|
||||
default:
|
||||
}
|
||||
m := spare
|
||||
spare = nil
|
||||
if m == nil {
|
||||
select {
|
||||
case lm := <-live:
|
||||
run(lm)
|
||||
continue
|
||||
case m = <-again:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
if liveWaits() {
|
||||
spare = m
|
||||
select {
|
||||
case lm := <-live:
|
||||
run(lm)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case lm := <-live: // came while the queue was asked
|
||||
spare = m
|
||||
run(lm)
|
||||
default:
|
||||
run(m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// itemsOf are the items the work queue holds of module's version in class (empty for any class), each with its subject.
|
||||
func (pq *publishQueue) itemsOf(ctx context.Context, module, version, class string) (map[string]versions.QueuedPublish, jetstream.Stream, error) {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
st, err := pq.js.Stream(ctx, b.Stream)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
cls := "*"
|
||||
if class != "" {
|
||||
cls = class
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.*."+cls+"."+versions.EscapeToken(module)+"."+versions.EscapeToken(version)))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out := map[string]versions.QueuedPublish{}
|
||||
for subj := range info.State.Subjects {
|
||||
raw, err := st.GetLastMsgForSubject(ctx, subj)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var q versions.QueuedPublish
|
||||
if json.Unmarshal(raw.Data, &q) == nil {
|
||||
out[subj] = q
|
||||
}
|
||||
}
|
||||
return out, st, nil
|
||||
}
|
||||
|
||||
// drop is builder_publish_drop (§793): the version's publish item is ended with who dropped it and why, so no item of it is ever
|
||||
// sent or put again; its items waiting on the work queue are removed, and one held is terminated.
|
||||
func (pq *publishQueue) drop(ctx context.Context, items *publishItems, module, version, by, why string) (string, error) {
|
||||
qs, st, err := pq.itemsOf(ctx, module, version, "")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
|
||||
}
|
||||
if err := items.endItem(ctx, module, version, fmt.Sprintf("dropped by %s: %s", by, why)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for subj := range qs {
|
||||
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
|
||||
return "", fmt.Errorf("the item %s is not terminated: %w", subj, err)
|
||||
}
|
||||
}
|
||||
for _, h := range items.settle(module, version, func(heldItem) bool { return true }) {
|
||||
_ = h.m.Term()
|
||||
}
|
||||
return fmt.Sprintf("the publish of %s %s is dropped: %d items terminated, and none is sent or put again", module, version, len(qs)), nil
|
||||
}
|
||||
|
||||
// first is builder_publish_first (§793): each again item of the version is put again as live, under the message id
|
||||
// `<holder>.live.<module>.<version>`, and the old terminated; a live id the queue holds within its window takes nothing, and the
|
||||
// old stays, said.
|
||||
func (pq *publishQueue) first(ctx context.Context, items *publishItems, module, version string) (string, error) {
|
||||
qs, st, err := pq.itemsOf(ctx, module, version, versions.PublishClassStart)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
|
||||
}
|
||||
if len(qs) == 0 {
|
||||
return "", fmt.Errorf("the work queue %s holds no again item of %s %s", versions.PublishesLocal, module, version)
|
||||
}
|
||||
var moved, kept []string
|
||||
for subj, q := range qs {
|
||||
q.Class = versions.PublishClassLive
|
||||
dup, err := pq.put(ctx, q)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s %s is not put again as live at %s: %w", module, version, q.Holder, err)
|
||||
}
|
||||
if dup {
|
||||
kept = append(kept, q.Holder)
|
||||
continue
|
||||
}
|
||||
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
|
||||
return "", fmt.Errorf("the again item of %s %s at %s is not terminated: %w", module, version, q.Holder, err)
|
||||
}
|
||||
moved = append(moved, q.Holder)
|
||||
}
|
||||
for _, h := range items.settle(module, version, func(h heldItem) bool { return h.q.Class == versions.PublishClassStart }) {
|
||||
_ = h.m.Term()
|
||||
}
|
||||
sort.Strings(moved)
|
||||
answer := fmt.Sprintf("%s %s is put first as live at %s, its again item terminated", module, version, strings.Join(moved, ", "))
|
||||
if len(kept) > 0 {
|
||||
sort.Strings(kept)
|
||||
answer += fmt.Sprintf("; at %s the work queue holds its live id within the duplicates window, so its again item stays", strings.Join(kept, ", "))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// endItem writes the version's publish item ended with answer, made when none stands: Try and take then leave it, and a start
|
||||
// asks it no more.
|
||||
func (p *publishItems) endItem(ctx context.Context, module, version, answer string) error {
|
||||
kv := p.reasks()
|
||||
if kv == nil {
|
||||
return errors.New("the re-ask state is not on the bus yet")
|
||||
}
|
||||
key := versions.PublishItemKey(module, version)
|
||||
for range 5 {
|
||||
r := build.ReAsk{V: 1, Why: "the build ended with its version unreleased", By: "the builder", At: p.now().UTC(), Mode: build.ModePublish, Module: module, Version: version}
|
||||
e, err := kv.Get(ctx, key)
|
||||
var rev uint64
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
case err != nil:
|
||||
return err
|
||||
default:
|
||||
if r, err = build.DecodeReAsk(e.Value()); err != nil {
|
||||
return err
|
||||
}
|
||||
rev = e.Revision()
|
||||
}
|
||||
r.Answer, r.Answered = answer, p.now().UTC()
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rev == 0 {
|
||||
_, err = kv.Create(ctx, key, data)
|
||||
} else {
|
||||
_, err = kv.Update(ctx, key, data, rev)
|
||||
}
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return errors.New("the publish item of " + module + " " + version + " changed five times while it was ended")
|
||||
}
|
||||
|
||||
// moveOld puts the items of the old state publish-queue on the work queue once at the start (§793): each under its message id, so
|
||||
// a move run twice puts each once within the duplicates window; then the bucket is dropped through the bus module (its tool
|
||||
// nats_drop, on the operator's warrant). A builder whose bus holds no such bucket moves nothing.
|
||||
func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string, rp *versions.Republisher, call toolCaller, logf func(string, ...any)) {
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logf("the old state %s cannot be read, so its items are not moved to the work queue %s: %v", versions.PublishQueueState, versions.PublishesLocal, err)
|
||||
return
|
||||
}
|
||||
keys, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
logf("the old state %s cannot be listed, so its items are not moved: %v", versions.PublishQueueState, err)
|
||||
return
|
||||
}
|
||||
var vals []json.RawMessage
|
||||
for k := range keys.Keys() {
|
||||
if e, err := kv.Get(ctx, k); err == nil {
|
||||
vals = append(vals, json.RawMessage(e.Value()))
|
||||
}
|
||||
}
|
||||
n, err := rp.MoveQueued(ctx, vals)
|
||||
if err != nil {
|
||||
logf("the old state %s is not dropped: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
|
||||
return
|
||||
}
|
||||
logf("the old state %s: %d of its %d items put on the work queue %s; it is dropped through the bus module", versions.PublishQueueState, n, len(vals), versions.PublishesLocal)
|
||||
if call == nil {
|
||||
return
|
||||
}
|
||||
if _, err := call(natsDropTool, map[string]any{"module": module, "local": versions.PublishQueueState,
|
||||
"why": "its items are on the builder's work queue " + versions.PublishesLocal + " (design 47 H2 section 793)"}); err != nil {
|
||||
logf("the old state %s is not dropped through the bus module: %v", versions.PublishQueueState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// natsDropTool is the bus module's drop of an entry no accepted manifest asks for (H-A-126), which the old state goes by.
|
||||
const natsDropTool = "nats.nats_drop"
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -89,6 +90,29 @@ func (k *itemKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOp
|
||||
return nil
|
||||
}
|
||||
|
||||
// itemMsgFake is an item pulled from the work queue: what take settled it with.
|
||||
type itemMsgFake struct {
|
||||
data []byte
|
||||
settled []string // ack, term, nak <delay>
|
||||
}
|
||||
|
||||
func (m *itemMsgFake) Data() []byte { return m.data }
|
||||
func (m *itemMsgFake) Ack() error { m.settled = append(m.settled, "ack"); return nil }
|
||||
func (m *itemMsgFake) Term() error { m.settled = append(m.settled, "term"); return nil }
|
||||
func (m *itemMsgFake) NakWithDelay(d time.Duration) error {
|
||||
m.settled = append(m.settled, "nak "+d.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
func queuedMsg(t *testing.T, module, version string) *itemMsgFake {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: version, Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: versions.PublishClassLive})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &itemMsgFake{data: raw}
|
||||
}
|
||||
|
||||
func decodeItem(t *testing.T, raw []byte) build.ReAsk {
|
||||
t.Helper()
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
@@ -98,21 +122,21 @@ func decodeItem(t *testing.T, raw []byte) build.ReAsk {
|
||||
return r
|
||||
}
|
||||
|
||||
// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a — a version's publish item is asked once, each try queues
|
||||
// the publish and writes the item again with one more try and the pause doubled up to its maximum, the release deletes it, and
|
||||
// after the setting's tries it ends with its words, raises the condition and stays; a version no longer reserved has it deleted.
|
||||
// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a, H2 §793 (H-A-440 (2), H-A-448) — a version's publish
|
||||
// item is asked once; its try puts the publish on the work queue and counts no try; each send counts one and a failed send hands
|
||||
// the item back with the pause doubled up to its maximum; after the setting's tries the item is terminated and ends with its words,
|
||||
// raises the condition and stays; an item whose artifacts are published is acknowledged; the release deletes it; a version no
|
||||
// longer reserved has it deleted.
|
||||
func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
now := time.Date(2026, 11, 17, 1, 0, 0, 0, time.UTC)
|
||||
var waited []time.Duration
|
||||
var queued, ended []string
|
||||
var queued, ended, sent []string
|
||||
reserved := true
|
||||
v := versions.ReservedVersion{Module: "m", Version: "0.1.2", Artifacts: map[string]string{"m:a": "sha256:a"}, BuiltAt: now}
|
||||
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: 30 * time.Second, PauseMax: 2 * time.Minute, Tries: 4},
|
||||
now: func() time.Time { return now }, logf: t.Logf,
|
||||
after: func(d time.Duration) <-chan time.Time {
|
||||
waited = append(waited, d)
|
||||
now = now.Add(d)
|
||||
c := make(chan time.Time, 1)
|
||||
c <- now
|
||||
@@ -126,7 +150,11 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
},
|
||||
republish: func(_ context.Context, m, ver string) ([]string, error) {
|
||||
queued = append(queued, m+" "+ver)
|
||||
return nil, errors.New("no holder")
|
||||
return []string{m + " " + ver}, nil
|
||||
},
|
||||
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
|
||||
sent = append(sent, q.Module+" "+q.Version)
|
||||
return versions.SendFate{Failed: "no holder answers"}
|
||||
},
|
||||
endedWith: func(_ context.Context, v versions.ReservedVersion, why string) {
|
||||
ended = append(ended, v.Version+": "+why)
|
||||
@@ -135,25 +163,34 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
if err := p.Ask(ctx, "m", "0.1.2"); err != nil || p.Ask(ctx, "m", "0.1.2") != nil || len(kv.vals) != 1 {
|
||||
t.Fatalf("asked: %v, %d items, want one item however often it is asked", err, len(kv.vals))
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
e, _ := kv.Get(ctx, key)
|
||||
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
|
||||
e, _ := kv.Get(ctx, key)
|
||||
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
|
||||
if len(queued) != 1 || decodeItem(t, kv.vals[key]).Tries != 0 {
|
||||
t.Fatalf("the try queued %v, tries %d: want the publish put once, no try counted", queued, decodeItem(t, kv.vals[key]).Tries)
|
||||
}
|
||||
var settled []string
|
||||
for range 5 {
|
||||
m := queuedMsg(t, "m", "0.1.2")
|
||||
p.take(ctx, m)
|
||||
settled = append(settled, m.settled...)
|
||||
}
|
||||
got := decodeItem(t, kv.vals[key])
|
||||
if len(queued) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") {
|
||||
t.Fatalf("queued %v, item %+v, ended %v: want four tries, then the end said with its words and the condition", queued, got, ended)
|
||||
if len(sent) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") {
|
||||
t.Fatalf("sent %v, item %+v, ended %v: want four sends, then the end said with its words and the condition", sent, got, ended)
|
||||
}
|
||||
if want := []time.Duration{30 * time.Second, time.Minute, 2 * time.Minute, 2 * time.Minute}; len(waited) != len(want) || waited[0] != want[0] || waited[1] != want[1] || waited[2] != want[2] || waited[3] != want[3] {
|
||||
t.Errorf("pauses before the tries and the end = %v, want %v (doubled, capped)", waited, want)
|
||||
if want := "[nak 30s nak 1m0s nak 2m0s nak 2m0s term]"; fmt.Sprint(settled) != want {
|
||||
t.Errorf("the items were settled %v, want %s (handed back with the pause doubled and capped, then terminated)", settled, want)
|
||||
}
|
||||
// An item whose artifacts are all published stays until its release is written, and is tried (A-456b).
|
||||
queued = nil
|
||||
m := queuedMsg(t, "m", "0.1.2")
|
||||
if p.take(ctx, m); fmt.Sprint(m.settled) != "[term]" || len(sent) != 4 {
|
||||
t.Errorf("an item of an ended publish item was settled %v, sent %d: want it terminated unsent", m.settled, len(sent))
|
||||
}
|
||||
// An item whose artifacts are all published is acknowledged unsent; its publish item stays until its release is written (A-456b).
|
||||
_ = p.Ask(ctx, "m", "0.3.0")
|
||||
pk := versions.PublishItemKey("m", "0.3.0")
|
||||
pe, _ := kv.Get(ctx, pk)
|
||||
p.Try(ctx, kv, pe, decodeItem(t, pe.Value()))
|
||||
if _, ok := kv.vals[pk]; !ok || len(queued) != 1 || decodeItem(t, kv.vals[pk]).Tries != 1 {
|
||||
t.Errorf("an item whose artifacts are published was deleted or not tried: %v, queued %v", kv.vals, queued)
|
||||
m = queuedMsg(t, "m", "0.3.0")
|
||||
if p.take(ctx, m); fmt.Sprint(m.settled) != "[ack]" || len(sent) != 4 || !kv.has(pk) {
|
||||
t.Errorf("an item whose artifacts are published was settled %v, sent %d, kept %v: want it acknowledged unsent, its publish item kept", m.settled, len(sent), kv.has(pk))
|
||||
}
|
||||
kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{}
|
||||
// At the start every reservation of a done build with no release is asked, one item each.
|
||||
@@ -194,7 +231,7 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
queued = nil
|
||||
_ = p.Ask(ctx, "m", "0.1.2")
|
||||
v.Withdrawn = map[string]string{"0.1.2": "fp"}
|
||||
e, _ := kv.Get(ctx, key)
|
||||
e, _ = kv.Get(ctx, key)
|
||||
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
|
||||
if len(kv.vals) != 0 || len(queued) != 0 {
|
||||
t.Errorf("a withdrawn reservation left its item %v, queued %v", kv.vals, queued)
|
||||
@@ -209,7 +246,8 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a — the three settings, 30 s, 30 min and 8 tries unless set, and a
|
||||
// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a, H2 H-A-446 (2) — the three settings, 30 s, 30 min and 5 tries
|
||||
// unless set, and a
|
||||
// content that is no number is refused naming its file.
|
||||
func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
@@ -222,7 +260,7 @@ func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
|
||||
}
|
||||
env := map[string]string{}
|
||||
got, err := publishSettingsOf(func(k string) string { return env[k] })
|
||||
if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 8 {
|
||||
if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 5 {
|
||||
t.Fatalf("defaults = %+v (%v)", got, err)
|
||||
}
|
||||
env["MESH_BUILDER_PUBLISH_PAUSE_FILE"], env["MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE"], env["MESH_BUILDER_PUBLISH_TRIES_FILE"] = write("p", "5\n"), write("m", "2"), write("t", "3")
|
||||
@@ -353,9 +391,12 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
asked, judged, marked, moved := 0, 0, 0, 0
|
||||
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: defaultPublishSettings, now: func() time.Time { return now }, logf: t.Logf,
|
||||
reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil },
|
||||
republish: func(context.Context, string, string) ([]string, error) { asked++; return nil, nil },
|
||||
unheard: func(context.Context) (int, error) { return c.unheard, c.countErr },
|
||||
reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil },
|
||||
send: func(context.Context, versions.QueuedPublish) versions.SendFate {
|
||||
asked++
|
||||
return versions.SendFate{Sent: true}
|
||||
},
|
||||
unheard: func(context.Context) (int, error) { return c.unheard, c.countErr },
|
||||
judge: func(context.Context, versions.ReservedVersion) (versions.Judged, error) {
|
||||
judged++
|
||||
return c.judged, nil
|
||||
@@ -374,20 +415,24 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := versions.PublishItemKey(v.Module, v.Version)
|
||||
e, _ := kv.Get(ctx, key)
|
||||
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
|
||||
m := queuedMsg(t, v.Module, v.Version)
|
||||
m.data = []byte(strings.ReplaceAll(string(m.data), `"node-tools:a"`, `"node-tools:program"`))
|
||||
p.take(ctx, m)
|
||||
if (asked == 1) != c.wantAsked || (marked == 1) != c.wantMark || (moved == 1) != c.wantMov {
|
||||
t.Fatalf("asked %d, marked %d, moved %d; want asked %v, marked %v, moved %v", asked, marked, moved, c.wantAsked, c.wantMark, c.wantMov)
|
||||
}
|
||||
if c.wantMov {
|
||||
if _, ok := kv.vals[key]; ok {
|
||||
t.Error("the moved version's old item is written again")
|
||||
if _, ok := kv.vals[key]; ok || fmt.Sprint(m.settled) != "[term]" {
|
||||
t.Errorf("the moved version's old item is written again, or its item settled %v, not terminated", m.settled)
|
||||
}
|
||||
return
|
||||
}
|
||||
got := decodeItem(t, kv.vals[key])
|
||||
if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || !got.NextTry.After(now) {
|
||||
t.Errorf("tries %d, next try %s: want a try spent only when asked, and the item written again with its pause", got.Tries, got.NextTry)
|
||||
if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || (c.wantAsked && !got.NextTry.After(now)) {
|
||||
t.Errorf("tries %d, next try %s: want a try spent only when sent, the item written with its pause", got.Tries, got.NextTry)
|
||||
}
|
||||
if !c.wantAsked && fmt.Sprint(m.settled) != "[nak 30s]" {
|
||||
t.Errorf("an item not sent was settled %v, want it handed back with the pause and no try spent", m.settled)
|
||||
}
|
||||
if c.unheard >= versions.UnheardBound && judged != 0 {
|
||||
t.Error("the registry is read while a published may be unheard")
|
||||
|
||||
@@ -65,7 +65,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi
|
||||
released(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
|
||||
go versionActed(ctx, acts, kv, e, r, logf)
|
||||
return
|
||||
@@ -208,6 +208,16 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue
|
||||
r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them"
|
||||
}
|
||||
}
|
||||
} else if r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst {
|
||||
act := acts.PublishDrop
|
||||
if r.Mode == build.ModePublishFirst {
|
||||
act = acts.PublishFirst
|
||||
}
|
||||
if act == nil {
|
||||
err = errors.New("this builder keeps no work queue of publishes")
|
||||
} else {
|
||||
r.Answer, err = act(ctx, r.Module, r.Version, r.By, r.Why)
|
||||
}
|
||||
} else if r.Mode == build.ModeReserveDone {
|
||||
if acts.ReserveDone == nil {
|
||||
err = errors.New("this builder keeps no reservations")
|
||||
|
||||
@@ -6,11 +6,8 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
@@ -21,21 +18,6 @@ import (
|
||||
|
||||
// The builder's publish again (design 47 G §588, A-73, A-76): the wiring of versions.Republisher to the bus.
|
||||
|
||||
// awaitedQueue is the state publish-queue as versions.QueueStore: bound once the bus holds it; its bytes are the cap's.
|
||||
type awaitedQueue struct{ awaitedReserved }
|
||||
|
||||
func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
st, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return st.Bytes(), nil
|
||||
}
|
||||
|
||||
// entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry.
|
||||
func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
|
||||
return func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
|
||||
@@ -114,21 +96,3 @@ func callWithin(ctx context.Context, call toolCaller, key string, body any) (jso
|
||||
func publishBody(p versions.Publish) gopackageregistry.PublishInput {
|
||||
return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version}
|
||||
}
|
||||
|
||||
// republishing starts the publish again once both its states are bound: the queue is taken up again (design 47 G §588). A version's
|
||||
// publish is asked when its build ends and at the start (A-457a), never waited on a running or manifest-reads-changed.
|
||||
type republishing struct {
|
||||
rp *versions.Republisher
|
||||
ready atomic.Int32
|
||||
}
|
||||
|
||||
func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) func(jetstream.KeyValue) {
|
||||
return func(jetstream.KeyValue) {
|
||||
if g.ready.Add(1) != 2 {
|
||||
return
|
||||
}
|
||||
if err := g.rp.Resume(ctx); err != nil {
|
||||
logf("the queued publishes are not taken up again: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,12 +45,15 @@ type reservations struct {
|
||||
// built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none.
|
||||
built func(ctx context.Context, module, version string) error
|
||||
released func(ctx context.Context, module, version string)
|
||||
filled atomic.Bool
|
||||
bounds func() jetstream.KeyValue
|
||||
found *findings
|
||||
bound time.Duration
|
||||
wake func(context.Context)
|
||||
logf func(string, ...any)
|
||||
// ended ends the publish item of a build withdrawn, superseded or dropped, its held items terminated; heard acknowledges the
|
||||
// held items of a version a published newly marked (design 47 H2 §793). Nil are none.
|
||||
itemEnded, itemHeard func(ctx context.Context, module, version string)
|
||||
filled atomic.Bool
|
||||
bounds func() jetstream.KeyValue
|
||||
found *findings
|
||||
bound time.Duration
|
||||
wake func(context.Context)
|
||||
logf func(string, ...any)
|
||||
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
|
||||
withdraw func(module, fingerprint, why string)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
@@ -146,8 +149,8 @@ func (rv *reservations) took(ctx context.Context, older, newer versions.Reserved
|
||||
} else {
|
||||
rv.logf("%s: the reservation of %s (build of %s, never released) is dropped: the build of %s takes the series over and reserves %s", newer.Module, older.Version, older.Fingerprint, newer.Fingerprint, newer.Version)
|
||||
}
|
||||
if rv.released != nil {
|
||||
rv.released(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b)
|
||||
if rv.itemEnded != nil {
|
||||
rv.itemEnded(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793)
|
||||
}
|
||||
rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series))
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module)
|
||||
@@ -350,6 +353,15 @@ func (rv *reservations) published(ctx context.Context, o versions.Outcome, inPro
|
||||
for _, m := range out.Marked {
|
||||
rv.sayPublished(ctx, m, o.Digest)
|
||||
}
|
||||
if rv.itemHeard != nil { // the publish items a published newly marked are acknowledged (§793)
|
||||
seen := map[string]bool{}
|
||||
for _, m := range out.Marked {
|
||||
if k := m.Reserved.Module + " " + m.Reserved.Version; !seen[k] {
|
||||
seen[k] = true
|
||||
rv.itemHeard(ctx, m.Reserved.Module, m.Reserved.Version)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayPublished says the seat mesh-builds' package-published of an artifact a published marked.
|
||||
@@ -372,8 +384,8 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ok && held && rv.released != nil {
|
||||
rv.released(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b)
|
||||
if ok && held && rv.itemEnded != nil {
|
||||
rv.itemEnded(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793)
|
||||
}
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s holds no reservation of %s", module, series)
|
||||
|
||||
@@ -457,6 +457,7 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) {
|
||||
var asked, ended []string
|
||||
rv.built = func(_ context.Context, m, v string) error { asked = append(asked, m+" "+v); return nil }
|
||||
rv.released = func(_ context.Context, m, v string) { ended = append(ended, m+" "+v) }
|
||||
rv.itemEnded = rv.released // a build withdrawn or superseded ends its publish item too (§793)
|
||||
registry := []string{"0.1.37"}
|
||||
rv.r.HeldNumbers = func(context.Context, string, string, []string) ([]string, error) { return registry, nil }
|
||||
rv.filled.Store(true)
|
||||
|
||||
@@ -5,6 +5,7 @@ go 1.26.0
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -74,6 +74,27 @@ const ModePublish = "publish"
|
||||
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
|
||||
const ModeRepublish = "republish"
|
||||
|
||||
// ModePublishDrop and ModePublishFirst are the Modes of a reasks value of builder_publish_drop and builder_publish_first (design
|
||||
// 47 H2 §793): Module and Version name the item; the builder terminates it, or puts it again as live and terminates the old.
|
||||
const (
|
||||
ModePublishDrop = "publish-drop"
|
||||
ModePublishFirst = "publish-first"
|
||||
)
|
||||
|
||||
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) or builder_publish_first (ModePublishFirst) writes
|
||||
// (§793): its key, `<mode>.<unix nanoseconds>` (a call is one value), and the value. Module, version and why are required.
|
||||
func NewPublishAct(mode, module, version, why, by string, at time.Time) (string, []byte, error) {
|
||||
module, version = strings.TrimSpace(module), strings.TrimSpace(version)
|
||||
if module == "" || version == "" {
|
||||
return "", nil, errors.New("give the item's module and version")
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why, in plain words (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: mode, Module: module, Version: version})
|
||||
return mode + "." + strconv.FormatInt(at.UnixNano(), 10), data, err
|
||||
}
|
||||
|
||||
// ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*).
|
||||
const ModeFloorAgain = "floor-again"
|
||||
|
||||
|
||||
@@ -27,6 +27,10 @@ type Acts struct {
|
||||
ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error)
|
||||
// DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not.
|
||||
DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error)
|
||||
// PublishDrop terminates one publish item, and PublishFirst puts it again as live and terminates the old (builder_publish_drop,
|
||||
// builder_publish_first, design 47 H2 §793): the answer, or why not.
|
||||
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
PublishFirst func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
}
|
||||
|
||||
// FloorAgain is builder_floor_again of name: refused, since the builder numbers nothing of its own and no claim asks a floor
|
||||
|
||||
@@ -17,22 +17,28 @@ import (
|
||||
|
||||
// **A reserved version of a done build is published by the builder until its release is written** (design 47 G §588, A-457a): the
|
||||
// publish is an item of the builder's re-ask state, asked when the build ends and at its start, tried with a growing pause and
|
||||
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has one queue of
|
||||
// versions, kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no
|
||||
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has its items on
|
||||
// the builder's work queue `publishes` (design 47 H2 §793), subject `publish.<holder>.<class>.<module>.<version>`, pulled by one
|
||||
// consumer per holder and class, the live class first. A send is the registry seat's `publish` with the digest and no
|
||||
// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. Nothing waits on a
|
||||
// node-engine `running` or `manifest-reads-changed` any more.
|
||||
|
||||
const (
|
||||
// PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB.
|
||||
// PublishQueueState is the builder's old state of the queues, whose items are put on the work queue once at the first start
|
||||
// of this release, then dropped through the bus module (design 47 H2 §793).
|
||||
PublishQueueState = "publish-queue"
|
||||
PublishQueueMiB = 4
|
||||
// PublishesLocal is the builder's work queue of publish items, asked for in its requires (§793, §820; H-A-448).
|
||||
PublishesLocal = "publishes"
|
||||
// ActedState holds, one key per subject, the last stream sequence the builder acted on; capped at ActedMiB.
|
||||
ActedState = "acted"
|
||||
ActedMiB = 1
|
||||
// PublishQueueTool is builder_publish_queue.
|
||||
PublishQueueTool = "builder_publish_queue"
|
||||
// QueueFull is what version-unreleased says of a version the queue's cap refused.
|
||||
// QueueFull is what version-unreleased says of a version the work queue's cap refused.
|
||||
QueueFull = "queue full"
|
||||
// PublishDropTool and PublishFirstTool are builder_publish_drop and builder_publish_first (§793).
|
||||
PublishDropTool = "builder_publish_drop"
|
||||
PublishFirstTool = "builder_publish_first"
|
||||
// NotStoredWords opens what a refused artifact is said as.
|
||||
NotStoredWords = "not stored: "
|
||||
// VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds
|
||||
@@ -40,11 +46,11 @@ const (
|
||||
VerbPublish = "publish"
|
||||
VerbVersions = "versions"
|
||||
// PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on;
|
||||
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's queue sends every
|
||||
// live one before any of the start, each class oldest first (design 47 H H-A-290 (1)).
|
||||
PublishClassLive = "build-ended-this-run"
|
||||
PublishClassStart = "asked-again-at-start"
|
||||
// PublishSendsAtOnce bounds the sends of one holder's queue that run at once (H-A-290 (3)).
|
||||
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled
|
||||
// live first, each class oldest first (design 47 H H-A-290 (1), H2 §793).
|
||||
PublishClassLive = "live"
|
||||
PublishClassStart = "again"
|
||||
// PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793).
|
||||
PublishSendsAtOnce = 4
|
||||
// PublishQueueListCap caps the queued versions and the publish items builder_publish_queue lists (H-A-290 (5)).
|
||||
PublishQueueListCap = 50
|
||||
@@ -77,9 +83,8 @@ func SeatOfArtifact(a meshcatalog.Artifact) (string, bool) {
|
||||
return seat, ok
|
||||
}
|
||||
|
||||
// QueuedPublish is the value of a publish-queue key `<holder module>.<module>.<version>` (each token escaped, §73): the
|
||||
// artifacts of the version this holder publishes, by reference with their digests, when it was queued and whether its
|
||||
// publish has begun.
|
||||
// QueuedPublish is one item of the work queue publishes (§793), and the value of a key of the old state publish-queue: the
|
||||
// artifacts of the version this holder publishes, by reference with their digests, and when it was queued.
|
||||
type QueuedPublish struct {
|
||||
V int `json:"v"`
|
||||
Holder string `json:"holder"`
|
||||
@@ -87,7 +92,6 @@ type QueuedPublish struct {
|
||||
Version string `json:"version"`
|
||||
Artifacts map[string]string `json:"artifacts"`
|
||||
Queued time.Time `json:"queued"`
|
||||
Begun bool `json:"begun,omitempty"`
|
||||
// Class is PublishClassLive or PublishClassStart (H-A-290 (1)); a value written before it had one is of the start.
|
||||
Class string `json:"class,omitempty"`
|
||||
}
|
||||
@@ -110,25 +114,31 @@ func sortQueued(qs []QueuedPublish) {
|
||||
})
|
||||
}
|
||||
|
||||
// QueueFilter is the filter of every queued version: a queue key has three tokens, which AllFilter (two) never lists on the bus.
|
||||
const QueueFilter = "*.*.*"
|
||||
|
||||
// QueueKey is the key of holder's queued version of module.
|
||||
// QueueKey is the key of holder's queued version of module in the old state publish-queue.
|
||||
func QueueKey(holder, module, version string) string {
|
||||
return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version)
|
||||
}
|
||||
|
||||
// PublishMsgID is the message id of holder's item of module's version in class (§793): the work queue takes one item of each
|
||||
// id within its duplicates window, so an item put twice is put once.
|
||||
func PublishMsgID(holder, class, module, version string) string {
|
||||
return EscapeToken(holder) + "." + class + "." + EscapeToken(module) + "." + EscapeToken(version)
|
||||
}
|
||||
|
||||
// PublishEvent is the subject of holder's item of module's version in class within the work queue publishes (§793):
|
||||
// `publish.<holder>.<class>.<module>.<version>`, each token escaped.
|
||||
func PublishEvent(holder, class, module, version string) string {
|
||||
return "publish." + PublishMsgID(holder, class, module, version)
|
||||
}
|
||||
|
||||
// ErrQueueFull is a put the work queue refused as full: said as QueueFull, and asked again at the item's next try (§820).
|
||||
var ErrQueueFull = errors.New("the work queue " + PublishesLocal + " is full")
|
||||
|
||||
// Acted is the value of an acted key: the last stream sequence acted on for its subject.
|
||||
type Acted struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
}
|
||||
|
||||
// QueueStore is the queue's bucket: compare-and-set, deletion and how many bytes it holds, for its cap.
|
||||
type QueueStore interface {
|
||||
ReserveStore
|
||||
Bytes(ctx context.Context) (uint64, error)
|
||||
}
|
||||
|
||||
// Publish is one artifact's publish again: what the registry seat's publish takes, without parts.
|
||||
type Publish struct {
|
||||
Seat, Kind, Name, Version, Digest, Fingerprint, Commit, Transfer string
|
||||
@@ -137,7 +147,9 @@ type Publish struct {
|
||||
// Republisher publishes reserved versions again (design 47 G §588).
|
||||
type Republisher struct {
|
||||
Reserved *Reserver
|
||||
Queue QueueStore
|
||||
// Put puts one item on the work queue publishes under its message id (PublishMsgID): duplicate says the queue holds that
|
||||
// id within its duplicates window and took nothing; a full queue is ErrQueueFull.
|
||||
Put func(ctx context.Context, q QueuedPublish) (duplicate bool, err error)
|
||||
// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
|
||||
Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
|
||||
// Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words.
|
||||
@@ -146,8 +158,6 @@ type Republisher struct {
|
||||
Unreleased func(ctx context.Context, v ReservedVersion, why string)
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
// Go starts the sending of a holder's queue; nil starts a goroutine.
|
||||
Go func(func())
|
||||
// Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat
|
||||
// of the assignments takes too, read from the builder's start (design 47 G A-416, A-416a); nil answers none.
|
||||
Holders func(seat string) []string
|
||||
@@ -158,11 +168,7 @@ type Republisher struct {
|
||||
// nil ends none.
|
||||
Ended func(ctx context.Context, module, version string)
|
||||
|
||||
mu sync.Mutex
|
||||
busy map[string]bool
|
||||
// relist marks a holder whose pass lists again before its next start-class send: a live publish was queued meanwhile
|
||||
// (H-A-291 (2)).
|
||||
relist map[string]bool
|
||||
mu sync.Mutex
|
||||
// lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock,
|
||||
// or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last.
|
||||
lastQueued time.Time
|
||||
@@ -292,18 +298,9 @@ func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[stri
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// drop deletes every queued key of module's version and ends its publish item: it never publishes (H-A-290 (2)), said with why.
|
||||
// drop ends module's version's publish item: it never publishes (H-A-290 (2)), said with why; its items on the work queue are
|
||||
// terminated when they are pulled (§793).
|
||||
func (r *Republisher) drop(ctx context.Context, module, version, why string) {
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
keys, err := r.Queue.KeysOf(ctx, "*."+EscapeToken(module)+"."+EscapeToken(version))
|
||||
if err != nil {
|
||||
r.logf("publish again: %s %s never publishes (%s), and its queued keys cannot be listed: %v", module, version, why, err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if err := r.Queue.Delete(ctx, k); err != nil {
|
||||
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
|
||||
}
|
||||
}
|
||||
r.end(ctx, module, version, why)
|
||||
}
|
||||
|
||||
@@ -338,13 +335,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
sort.SliceStable(vs, func(i, j int) bool { return vs[i].At.After(vs[j].At) })
|
||||
var queued []string
|
||||
var errs []error
|
||||
started := map[string]bool{}
|
||||
// Every holder given a version is drained, whatever a later version met: a version queued and never drained waits forever.
|
||||
defer func() {
|
||||
for holder := range started {
|
||||
r.drainLater(ctx, holder)
|
||||
}
|
||||
}()
|
||||
for _, v := range vs {
|
||||
if max > 0 && len(queued) >= max {
|
||||
break
|
||||
@@ -396,7 +386,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
return queued, err
|
||||
}
|
||||
did = did || ok
|
||||
started[holder] = true
|
||||
}
|
||||
if did {
|
||||
queued = append(queued, v.Module+" "+v.Version)
|
||||
@@ -405,42 +394,25 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
return queued, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// enqueue puts one holder's artifacts of a version in its queue: not again when it is queued already, and not past the
|
||||
// section's cap (said as queue full).
|
||||
// enqueue puts one holder's artifacts of a version on the work queue under its message id: not again while the queue holds that
|
||||
// id (its duplicates window), which stands queued, and not past the queue's cap (said as queue full).
|
||||
func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder string, arts map[string]string) (bool, error) {
|
||||
key := QueueKey(holder, v.Module, v.Version)
|
||||
if _, _, found, err := r.Queue.Get(ctx, key); err != nil {
|
||||
return false, err
|
||||
} else if found {
|
||||
r.logf("publish again: %s %s is queued at %s already", v.Module, v.Version, holder)
|
||||
return false, nil
|
||||
}
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)})
|
||||
used, err := r.Queue.Bytes(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the size of the state %s cannot be read: %w", PublishQueueState, err)
|
||||
}
|
||||
if used+uint64(len(raw)) > PublishQueueMiB<<20 {
|
||||
q := QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)}
|
||||
dup, err := r.Put(ctx, q)
|
||||
switch {
|
||||
case errors.Is(err, ErrQueueFull):
|
||||
r.logf("publish again: %s %s is not queued at %s: %v; its publish item asks it again at its next try", v.Module, v.Version, holder, err)
|
||||
if r.Unreleased != nil {
|
||||
r.Unreleased(ctx, v, QueueFull)
|
||||
}
|
||||
return false, nil
|
||||
case err != nil:
|
||||
return false, fmt.Errorf("%s %s cannot be queued at %s: %w", v.Module, v.Version, holder, err)
|
||||
case dup:
|
||||
r.logf("publish again: %s %s is queued at %s already (class %s): the work queue holds its message id", v.Module, v.Version, holder, q.Class)
|
||||
return true, nil
|
||||
}
|
||||
if _, err := r.Queue.Create(ctx, key, raw); err != nil {
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("%s cannot be queued: %w", key, err)
|
||||
}
|
||||
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), r.classOf(v))
|
||||
if r.classOf(v) == PublishClassLive { // a pass sending a start list lists again, so this one goes next (H-A-291 (2))
|
||||
r.mu.Lock()
|
||||
if r.relist == nil {
|
||||
r.relist = map[string]bool{}
|
||||
}
|
||||
r.relist[holder] = true
|
||||
r.mu.Unlock()
|
||||
}
|
||||
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), q.Class)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -537,15 +509,15 @@ func PublishItemKey(module, version string) string {
|
||||
return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version)
|
||||
}
|
||||
|
||||
// AnswerQueue is builder_publish_queue over the three states as the tools bundle reads them (H-A-290 (5)): each state read in
|
||||
// bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the
|
||||
// AnswerQueue is builder_publish_queue over the work queue publishes (its items, read in pages through the tool runtime, §793,
|
||||
// H-A-448) and the two states as the tools bundle reads them (H-A-290 (5)): each read in bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the
|
||||
// listed queue to one module's (one version's), so an item's class can be read.
|
||||
func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) {
|
||||
out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}}
|
||||
vals, c, err := queue.values(PublishQueueState)
|
||||
vals, c, err := queue.values(PublishesLocal)
|
||||
out.capped(c)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the state %s cannot be read: %w", PublishQueueState, err)
|
||||
return out, fmt.Errorf("the work queue %s cannot be read: %w", PublishesLocal, err)
|
||||
}
|
||||
counts := map[[2]string]*QueueCount{}
|
||||
var listed []QueuedPublish
|
||||
@@ -637,13 +609,21 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e
|
||||
}
|
||||
|
||||
// PublishQueueDescription is builder_publish_queue's description.
|
||||
const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588, H H-A-290): " +
|
||||
"how many each registry seat holder's queue holds per class (build-ended-this-run, sent first, or asked-again-at-start) with the oldest and newest " +
|
||||
"queue time; the first 50 in the order they are sent, each with its class, its artifacts' digests, when it was queued and whether its publish has " +
|
||||
"begun (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
|
||||
const PublishQueueDescription = "The versions the builder will publish again (its work queue " + PublishesLocal + ", design 47 G section 588, H2 section 793): " +
|
||||
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, pulled first, or again, asked again at a start) " +
|
||||
"with the oldest and newest queue time; the first 50 in the order they are pulled, each with its class, its artifacts' digests and when it was " +
|
||||
"queued (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
|
||||
"sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " +
|
||||
"tries and next try, and why one ended after its tries. Each state is read in one page; a state past it is answered capped, naming the " +
|
||||
"keys read and the state's size (H-A-293). Read-only."
|
||||
"tries and next try, and why one ended (its tries spent, or dropped). A read past its pages is answered capped, naming the items read and the " +
|
||||
"queue's size (H-A-293). Read-only."
|
||||
|
||||
// PublishDropDescription and PublishFirstDescription are builder_publish_drop's and builder_publish_first's (§793).
|
||||
const PublishDropDescription = "Drop one version's publish (design 47 H2 section 793): its items on the work queue " + PublishesLocal + " are " +
|
||||
"terminated at every holder, and its publish item ended with who dropped it and why, so it is never sent or put again; builder_publish_again " +
|
||||
"asks it anew. A hand act, recorded in the hand-act log first."
|
||||
const PublishFirstDescription = "Put one version's publish first (design 47 H2 section 793): each of its again items on the work queue " +
|
||||
PublishesLocal + " is put again as live, under the message id <holder>.live.<module>.<version>, pulled before every again item, and the old " +
|
||||
"terminated. A hand act, recorded in the hand-act log first."
|
||||
|
||||
// PublishAgainDescription is builder_publish_again's description.
|
||||
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
|
||||
@@ -652,17 +632,14 @@ const PublishAgainDescription = "Publish reserved versions again (design 47 G se
|
||||
"with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " +
|
||||
"recorded in the hand-act log."
|
||||
|
||||
// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588): a version whose reservation
|
||||
// was withdrawn or taken over is dropped with its publish item ended (H-A-290 (2)), and one of the live class whose build ended
|
||||
// before this run is of the start's now (H-A-290 (1)). It reads the reservations once.
|
||||
func (r *Republisher) Resume(ctx context.Context) error {
|
||||
keys, err := r.Queue.KeysOf(ctx, QueueFilter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
|
||||
}
|
||||
// MoveQueued puts the values of the old state publish-queue on the work queue, once (§793): each under its message id, so a
|
||||
// move run twice puts each once within the duplicates window; its class as of this run (a build ended before it is of the
|
||||
// start's, H-A-290 (1)); a version whose reservation was withdrawn, taken over or superseded is not put, its publish item
|
||||
// ended (H-A-290 (2), H-A-291 (1)). It reads the reservations once, and answers how many it put.
|
||||
func (r *Republisher) MoveQueued(ctx context.Context, vals []json.RawMessage) (int, error) {
|
||||
all, err := r.Reserved.All(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the queued publishes are not taken up: %w", err)
|
||||
return 0, fmt.Errorf("the queued publishes are not moved: %w", err)
|
||||
}
|
||||
reserved := make(map[string]ReservedVersion, len(all))
|
||||
for _, v := range all {
|
||||
@@ -670,16 +647,12 @@ func (r *Republisher) Resume(ctx context.Context) error {
|
||||
}
|
||||
retired, err := r.Reserved.Retired(ctx, all)
|
||||
if err != nil {
|
||||
r.logf("publish again: the superseded builds are not retired at the start: %v", err)
|
||||
r.logf("publish again: the superseded builds are not retired at the move: %v", err)
|
||||
}
|
||||
holders := map[string]bool{}
|
||||
for _, k := range keys {
|
||||
raw, rev, found, err := r.Queue.Get(ctx, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
put := 0
|
||||
for _, raw := range vals {
|
||||
var q QueuedPublish
|
||||
if json.Unmarshal(raw, &q) != nil {
|
||||
if json.Unmarshal(raw, &q) != nil || q.Holder == "" {
|
||||
continue
|
||||
}
|
||||
v, ok := reserved[q.Module+" "+q.Version]
|
||||
@@ -688,126 +661,38 @@ func (r *Republisher) Resume(ctx context.Context) error {
|
||||
why = retired[q.Module+" "+q.Version]
|
||||
}
|
||||
if why != "" {
|
||||
if err := r.Queue.Delete(ctx, k); err != nil {
|
||||
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
|
||||
}
|
||||
r.end(ctx, q.Module, q.Version, why)
|
||||
continue
|
||||
}
|
||||
if class := r.classOf(v); q.Class != class {
|
||||
q.Class = class
|
||||
if raw, err := json.Marshal(q); err == nil {
|
||||
if _, err := r.Queue.Update(ctx, k, raw, rev); err != nil {
|
||||
r.logf("%s keeps its class %s at the start: %v", k, q.Class, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
holders[q.Holder] = true
|
||||
}
|
||||
for h := range holders {
|
||||
r.drainLater(ctx, h)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Republisher) drainLater(ctx context.Context, holder string) {
|
||||
r.mu.Lock()
|
||||
if r.busy == nil {
|
||||
r.busy = map[string]bool{}
|
||||
}
|
||||
if r.busy[holder] {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.busy[holder] = true
|
||||
r.mu.Unlock()
|
||||
run := func() {
|
||||
defer func() { r.mu.Lock(); delete(r.busy, holder); r.mu.Unlock() }()
|
||||
r.Drain(ctx, holder)
|
||||
}
|
||||
if r.Go != nil {
|
||||
r.Go(run)
|
||||
return
|
||||
}
|
||||
go run()
|
||||
}
|
||||
|
||||
// Drain sends holder's queued versions in passes (H-A-290 (3)): a pass lists the holder's keys once, reads each once, and sends
|
||||
// them in their order (the live class first, each class oldest first), PublishSendsAtOnce at a time; the keys are listed again
|
||||
// only when the pass is spent, or when a live publish was queued while it sends a start list: then the sends in flight end and
|
||||
// the keys are listed again, so the live one goes next (H-A-291 (2)); a start-class one queued meanwhile waits for the pass's end.
|
||||
// Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said.
|
||||
func (r *Republisher) Drain(ctx context.Context, holder string) {
|
||||
for ctx.Err() == nil {
|
||||
r.mu.Lock()
|
||||
delete(r.relist, holder)
|
||||
r.mu.Unlock()
|
||||
keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*")
|
||||
q.Class = r.classOf(v)
|
||||
dup, err := r.Put(ctx, q)
|
||||
if err != nil {
|
||||
r.logf("the queue of %s cannot be listed: %v", holder, err)
|
||||
return
|
||||
return put, fmt.Errorf("%s %s is not moved to the work queue %s: %w", q.Module, q.Version, PublishesLocal, err)
|
||||
}
|
||||
items := make([]QueuedPublish, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found {
|
||||
var q QueuedPublish
|
||||
if json.Unmarshal(raw, &q) == nil {
|
||||
items = append(items, q)
|
||||
}
|
||||
}
|
||||
if !dup {
|
||||
put++
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return
|
||||
}
|
||||
sortQueued(items)
|
||||
sem := make(chan struct{}, PublishSendsAtOnce)
|
||||
var wg sync.WaitGroup
|
||||
for _, q := range items {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
sem <- struct{}{}
|
||||
r.mu.Lock()
|
||||
again := q.Class != PublishClassLive && r.relist[holder]
|
||||
r.mu.Unlock()
|
||||
if again {
|
||||
<-sem
|
||||
r.logf("publish again: %s's pass lists again after its sends in flight: a live publish was queued", holder)
|
||||
break
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer func() { <-sem; wg.Done() }()
|
||||
r.send(ctx, q)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
return put, nil
|
||||
}
|
||||
|
||||
// send publishes one queued version's artifacts and says the outcome, then deletes its key. Every call it makes honours the one
|
||||
// bound it is given, UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why
|
||||
// (H-A-290 (4)).
|
||||
func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
|
||||
key := QueueKey(q.Holder, q.Module, q.Version)
|
||||
defer func() {
|
||||
if err := r.Queue.Delete(context.WithoutCancel(ctx), key); err != nil {
|
||||
r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err)
|
||||
}
|
||||
}()
|
||||
// SendFate is what SendQueued did with one item (§793): Sent, every artifact sent (its item acknowledged when its publish is
|
||||
// heard or confirmed); Ended, why the version never publishes (its item terminated); Failed, the holder's or the call's words
|
||||
// (its item handed back with the pause).
|
||||
type SendFate struct {
|
||||
Sent bool
|
||||
Ended, Failed string
|
||||
}
|
||||
|
||||
// SendQueued publishes one item's artifacts and says the outcome. Every call it makes honours the one bound it is given,
|
||||
// UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why (H-A-290 (4)).
|
||||
func (r *Republisher) SendQueued(ctx context.Context, q QueuedPublish) SendFate {
|
||||
bound, cancel := context.WithTimeout(ctx, UnreleasedBound)
|
||||
defer cancel()
|
||||
if !q.Begun {
|
||||
q.Begun = true
|
||||
raw, _ := json.Marshal(q)
|
||||
if _, rev, found, err := r.Queue.Get(bound, key); err == nil && found {
|
||||
_, _ = r.Queue.Update(bound, key, raw, rev)
|
||||
}
|
||||
}
|
||||
all, err := r.Reserved.All(bound)
|
||||
if err != nil {
|
||||
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; its publish item asks it again", q.Module, q.Version, q.Holder, err)
|
||||
return
|
||||
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; it is handed back", q.Module, q.Version, q.Holder, err)
|
||||
return SendFate{Failed: "its reservation cannot be read: " + err.Error()}
|
||||
}
|
||||
var v ReservedVersion
|
||||
found := false
|
||||
@@ -830,43 +715,64 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
|
||||
}
|
||||
if why != "" {
|
||||
r.end(ctx, q.Module, q.Version, why)
|
||||
return
|
||||
return SendFate{Ended: why}
|
||||
}
|
||||
e, err := r.Entry(bound, v.Fingerprint)
|
||||
if err != nil {
|
||||
r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err))
|
||||
return
|
||||
words := fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err)
|
||||
r.say(ctx, v, words)
|
||||
return SendFate{Failed: words}
|
||||
}
|
||||
missing := map[string]bool{}
|
||||
for _, ref := range v.Missing() {
|
||||
missing[ref] = true
|
||||
}
|
||||
refs := make([]string, 0, len(q.Artifacts))
|
||||
for ref := range q.Artifacts {
|
||||
if _, no := v.Unstored[ref]; no || !missing[ref] {
|
||||
continue // published already, or refused as not stored: said once, never sent again
|
||||
}
|
||||
refs = append(refs, ref)
|
||||
}
|
||||
sort.Strings(refs)
|
||||
if len(refs) == 0 {
|
||||
return SendFate{Ended: "none of its artifacts is left to send: each is published or refused as not stored"}
|
||||
}
|
||||
var failed []string
|
||||
sent := 0
|
||||
for _, ref := range refs {
|
||||
seat, name, kind, ok := publishTarget(e, q.Module, ref)
|
||||
if !ok {
|
||||
r.say(ctx, v, ref+": the catalog entry names no registry seat for it")
|
||||
return
|
||||
return SendFate{Failed: ref + ": the catalog entry names no registry seat for it"}
|
||||
}
|
||||
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
|
||||
Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))})
|
||||
if err == nil {
|
||||
r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref])
|
||||
sent++
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
|
||||
words := err.Error()
|
||||
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
|
||||
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
|
||||
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
|
||||
} else if ok {
|
||||
r.say(ctx, v, ref+" "+NotStoredWords+words)
|
||||
}
|
||||
continue
|
||||
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
|
||||
words := err.Error()
|
||||
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
|
||||
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
|
||||
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
|
||||
} else if ok {
|
||||
r.say(ctx, v, ref+" "+NotStoredWords+words)
|
||||
}
|
||||
r.say(ctx, v, ref+": "+words)
|
||||
continue
|
||||
}
|
||||
r.say(ctx, v, ref+": "+words)
|
||||
failed = append(failed, ref+": "+words)
|
||||
}
|
||||
switch {
|
||||
case len(failed) > 0:
|
||||
return SendFate{Failed: strings.Join(failed, "; ")}
|
||||
case sent == 0:
|
||||
return SendFate{Ended: "each artifact left was refused as not stored"}
|
||||
}
|
||||
return SendFate{Sent: true}
|
||||
}
|
||||
|
||||
// publishTarget is the registry seat, the name and the kind module's artifact ref is published as, from its catalog entry; ok is
|
||||
|
||||
@@ -9,51 +9,40 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
type queueMem struct {
|
||||
*reserveMem
|
||||
bytes uint64
|
||||
// listings counts the listings of one holder's keys (a filter `<holder>.*.*`), what a drain's pass lists.
|
||||
listings atomic.Int32
|
||||
}
|
||||
|
||||
func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil }
|
||||
|
||||
func (q *queueMem) KeysOf(ctx context.Context, filter string) ([]string, error) {
|
||||
if strings.HasSuffix(filter, ".*.*") && filter != QueueFilter {
|
||||
q.listings.Add(1)
|
||||
}
|
||||
return q.reserveMem.KeysOf(ctx, filter)
|
||||
}
|
||||
|
||||
// republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by
|
||||
// the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record.
|
||||
// the Go seat), one holder module `oci` that claims the oci seat, a work queue that records its puts (deduplicating by message
|
||||
// id when window is set, refusing when full) and sends each put at once as its pull would, and a Send and a Unreleased that
|
||||
// record.
|
||||
type republishBed struct {
|
||||
*Republisher
|
||||
reserved *reserveMem
|
||||
queue *queueMem
|
||||
now time.Time
|
||||
mu sync.Mutex // sent and said, which a holder's sends at once write
|
||||
mu sync.Mutex // sent and said, which sends at once write
|
||||
sent []Publish
|
||||
said []string
|
||||
ended []string
|
||||
refuse string // the words Send fails with, empty for none
|
||||
// holders are the registry seats' holders the seat-holders view names (design 47 G A-416).
|
||||
holders map[string][]string
|
||||
// puts are the items the work queue took; window deduplicates by message id; full refuses every put; held keeps the puts
|
||||
// unsent.
|
||||
puts []QueuedPublish
|
||||
ids map[string]bool
|
||||
window, full bool
|
||||
held bool
|
||||
}
|
||||
|
||||
func newRepublishBed(t *testing.T) *republishBed {
|
||||
t.Helper()
|
||||
b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)}
|
||||
b := &republishBed{reserved: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC), ids: map[string]bool{}}
|
||||
res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
|
||||
b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Now: func() time.Time { return b.now }, Go: func(f func()) { f() },
|
||||
b.Republisher = &Republisher{Reserved: res, Now: func() time.Time { return b.now },
|
||||
Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) {
|
||||
if fp == "src2:holder" {
|
||||
return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil
|
||||
@@ -83,6 +72,21 @@ func newRepublishBed(t *testing.T) *republishBed {
|
||||
b.ended = append(b.ended, module+" "+version)
|
||||
},
|
||||
Logf: t.Logf}
|
||||
b.Put = func(ctx context.Context, q QueuedPublish) (bool, error) {
|
||||
if b.full {
|
||||
return false, fmt.Errorf("%w: maximum bytes exceeded", ErrQueueFull)
|
||||
}
|
||||
id := PublishMsgID(q.Holder, q.Class, q.Module, q.Version)
|
||||
if b.window && b.ids[id] {
|
||||
return true, nil
|
||||
}
|
||||
b.ids[id] = true
|
||||
b.puts = append(b.puts, q)
|
||||
if !b.held {
|
||||
b.SendQueued(ctx, q)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
b.Holders = func(seat string) []string { return b.holders[seat] }
|
||||
return b
|
||||
}
|
||||
@@ -125,8 +129,8 @@ func TestRepublish(t *testing.T) {
|
||||
p.Transfer != "republish-m-"+strings.ReplaceAll(v.Version, ".", "-")+"-prog" || strings.ContainsAny(p.Transfer, "/. *>") {
|
||||
t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p)
|
||||
}
|
||||
if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 {
|
||||
t.Errorf("a key stays in the queue after its outcome: %v", keys)
|
||||
if len(b.puts) != 1 || b.puts[0].Holder != "oci" || b.puts[0].Class != PublishClassLive {
|
||||
t.Errorf("the work queue took %+v, want one live item at oci", b.puts)
|
||||
}
|
||||
})
|
||||
t.Run("an entry that cannot be read stops neither the other versions nor the drain", func(t *testing.T) {
|
||||
@@ -173,22 +177,42 @@ func TestRepublish(t *testing.T) {
|
||||
t.Run("a queue past its cap says queue full", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
b.reserve(t, "m")
|
||||
b.queue.bytes = PublishQueueMiB << 20
|
||||
b.full = true
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
_, _ = b.Republish(ctx, "", "")
|
||||
if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) {
|
||||
t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said)
|
||||
}
|
||||
})
|
||||
t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) {
|
||||
t.Run("an item put twice within the duplicates window is put once, and stands queued", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true})
|
||||
if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil {
|
||||
b.window, b.held = true, true
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
q1, _ := b.Republish(ctx, "m", v.Version)
|
||||
q2, _ := b.Republish(ctx, "m", v.Version)
|
||||
if len(b.puts) != 1 || len(q1) != 1 || len(q2) != 1 {
|
||||
t.Errorf("puts %d, queued %v then %v: want one item, queued both times", len(b.puts), q1, q2)
|
||||
}
|
||||
})
|
||||
t.Run("the move puts each item of the old state once, its class as of this run, a superseded one not", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
w := b.reserve(t, "n")
|
||||
b.Started = b.now
|
||||
b.window, b.held = true, true
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Class: "build-ended-this-run"})
|
||||
gone, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "n", Version: w.Version, Artifacts: map[string]string{"n:prog": "sha256:aa"}, Queued: b.now})
|
||||
if _, _, err := b.Reserved.Withdraw(ctx, "n", w.Version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.Resume(ctx); err != nil || len(b.sent) != 1 {
|
||||
t.Errorf("resume sent %d (%v), want the begun version sent anew", len(b.sent), err)
|
||||
for range 2 {
|
||||
if _, err := b.MoveQueued(ctx, []json.RawMessage{raw, gone}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(b.puts) != 1 || b.puts[0].Class != PublishClassStart || fmt.Sprint(b.ended) != "[n "+w.Version+" n "+w.Version+"]" {
|
||||
t.Errorf("puts %+v, ended %v: want m once as again (its build ended before this run), n's item ended", b.puts, b.ended)
|
||||
}
|
||||
})
|
||||
t.Run("after a fresh start the tool queues at the seat's holder, no running heard", func(t *testing.T) {
|
||||
@@ -262,98 +286,31 @@ func TestTheRegistriesRecordSaysWhetherAVersionIsHeld(t *testing.T) {
|
||||
}
|
||||
|
||||
// queueAt writes a queued version of module m at holder oci directly, as a start finds it.
|
||||
func (b *republishBed) queueAt(t *testing.T, v ReservedVersion, class string, at time.Time) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class})
|
||||
if _, err := b.queue.Create(context.Background(), QueueKey("oci", v.Module, v.Version), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
func (b *republishBed) queueAt(v ReservedVersion, class string, at time.Time) QueuedPublish {
|
||||
return QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class}
|
||||
}
|
||||
|
||||
// TestAHoldersQueueSendsLiveFirstInPasses: design 47 H H-A-290 (1)-(3) — a publish whose build ended in this run is sent before
|
||||
// any of a build that ended before the start, each class oldest first; a pass lists the holder's keys once and sends them, at most
|
||||
// four at a time; a build whose reservation was withdrawn or taken over never publishes, its key dropped and its publish item ended,
|
||||
// at enqueue, at the pass and at the start.
|
||||
func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
|
||||
// TestAQueuedItemIsSentWhileItsBuildStands: design 47 H H-A-290 (2), H-A-291 (1), H2 §793 — an item is put at the holder in the
|
||||
// class of its build (live when it ended in this run); a build whose reservation was withdrawn, taken over or superseded by a newer
|
||||
// patch or release of its series never publishes, its publish item ended, at the put and at the send; an item whose artifacts are
|
||||
// all published or refused as not stored ends unsent.
|
||||
func TestAQueuedItemIsSentWhileItsBuildStands(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
t.Run("the live class first, at most four sends at once", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
var old []ReservedVersion
|
||||
for _, m := range []string{"o1", "o2", "o3", "o4", "o5"} {
|
||||
old = append(old, b.reserve(t, m))
|
||||
}
|
||||
b.Started = b.now
|
||||
b.reserve(t, "live")
|
||||
for i, v := range old {
|
||||
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
|
||||
}
|
||||
b.now = b.now.Add(time.Hour) // the live one is queued last, after every old one
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
release := make(chan struct{})
|
||||
var arrived []string
|
||||
var inflight, most int
|
||||
send := b.Send
|
||||
b.Send = func(ctx context.Context, p Publish) error {
|
||||
b.mu.Lock()
|
||||
arrived = append(arrived, p.Name)
|
||||
inflight++
|
||||
most = max(most, inflight)
|
||||
b.mu.Unlock()
|
||||
<-release
|
||||
b.mu.Lock()
|
||||
inflight--
|
||||
b.mu.Unlock()
|
||||
return send(ctx, p)
|
||||
}
|
||||
b.Go = func(f func()) { go f() }
|
||||
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
|
||||
t.Fatalf("queued %v (%v), want the live version", q, err)
|
||||
}
|
||||
synctest.Wait() // every send the bound lets run waits on release
|
||||
b.mu.Lock()
|
||||
first := slices.Clone(arrived)
|
||||
b.mu.Unlock()
|
||||
if len(first) != PublishSendsAtOnce || !slices.Contains(first, "live/prog") {
|
||||
t.Errorf("the sends running at once are %v: want %d, the live one among them", first, PublishSendsAtOnce)
|
||||
}
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
if len(b.sent) != 6 || most != PublishSendsAtOnce {
|
||||
t.Errorf("sent %d, at most %d at once: want all six, %d at once", len(b.sent), most, PublishSendsAtOnce)
|
||||
}
|
||||
})
|
||||
})
|
||||
t.Run("a pass lists the keys once for all its sends", func(t *testing.T) {
|
||||
t.Run("a build that ended before this run is put as again, one after it as live", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
var vs []ReservedVersion
|
||||
for i := range 6 {
|
||||
vs = append(vs, b.reserve(t, fmt.Sprintf("m%d", i)))
|
||||
}
|
||||
for i, v := range vs {
|
||||
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
|
||||
}
|
||||
b.Drain(ctx, "oci")
|
||||
if n := b.queue.listings.Load(); len(b.sent) != 6 || n != 2 {
|
||||
t.Errorf("sent %d with %d listings, want six sends from one pass and one listing that finds it spent", len(b.sent), n)
|
||||
}
|
||||
})
|
||||
t.Run("a start makes a live key of a build before it a start key", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
old := b.reserve(t, "o")
|
||||
b.Started = b.now
|
||||
b.queueAt(t, v, PublishClassLive, b.now)
|
||||
b.Go = func(func()) {} // the start's drain is not run here
|
||||
if err := b.Resume(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _, _, _ := b.queue.Get(ctx, QueueKey("oci", "m", v.Version))
|
||||
var q QueuedPublish
|
||||
if json.Unmarshal(raw, &q) != nil || q.Class != PublishClassStart {
|
||||
t.Errorf("the key after the start is %s, want the class %s: its build did not end in this run", raw, PublishClassStart)
|
||||
live := b.reserve(t, "l")
|
||||
b.held = true
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
_, _ = b.Republish(ctx, "o", old.Version)
|
||||
_, _ = b.Republish(ctx, "l", live.Version)
|
||||
if len(b.puts) != 2 || b.puts[0].Class != PublishClassStart || b.puts[1].Class != PublishClassLive ||
|
||||
PublishEvent("oci", PublishClassLive, "l", live.Version) != "publish.oci.live.l.0=2E1=2E0" {
|
||||
t.Errorf("puts %+v, want o again and l live, on publish.<holder>.<class>.<module>.<version>", b.puts)
|
||||
}
|
||||
})
|
||||
for _, at := range []string{"enqueue", "pass", "start"} {
|
||||
for _, at := range []string{"put", "send"} {
|
||||
t.Run("a withdrawn build never publishes, at the "+at, func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
@@ -361,105 +318,34 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
|
||||
if _, ok, err := b.Reserved.Withdraw(ctx, "m", v.Version); err != nil || !ok {
|
||||
t.Fatalf("withdrawn %v (%v)", ok, err)
|
||||
}
|
||||
switch at {
|
||||
case "enqueue":
|
||||
b.queueAt(t, v, PublishClassLive, b.now)
|
||||
b.Go = func(func()) {} // a key queued before the withdrawal is dropped by the enqueue itself
|
||||
if at == "put" {
|
||||
if q, _ := b.Republish(ctx, "m", v.Version); len(q) != 0 {
|
||||
t.Errorf("queued %v", q)
|
||||
}
|
||||
case "pass":
|
||||
b.queueAt(t, v, PublishClassLive, b.now)
|
||||
b.Drain(ctx, "oci")
|
||||
case "start":
|
||||
b.queueAt(t, v, PublishClassLive, b.now)
|
||||
b.Go = func(func()) {}
|
||||
if err := b.Resume(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" {
|
||||
t.Errorf("the send of a withdrawn build answered %+v, want it ended", f)
|
||||
}
|
||||
keys, _ := b.queue.KeysOf(ctx, QueueFilter)
|
||||
if len(b.sent) != 0 || len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("sent %v, keys %v, items ended %v: want nothing sent, the key dropped and m %s's item ended", b.sent, keys, b.ended, v.Version)
|
||||
if len(b.sent) != 0 || len(b.puts) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("sent %v, puts %v, items ended %v: want nothing sent or put and m %s's item ended", b.sent, b.puts, b.ended, v.Version)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("a live publish queued mid-pass goes after the sends in flight, before the rest of the start list", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
var old []ReservedVersion
|
||||
for _, m := range []string{"o1", "o2", "o3", "o4", "o5", "o6"} {
|
||||
old = append(old, b.reserve(t, m))
|
||||
}
|
||||
b.Started = b.now
|
||||
b.reserve(t, "live")
|
||||
for i, v := range old {
|
||||
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
|
||||
}
|
||||
b.now = b.now.Add(time.Hour)
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
first, then := make(chan struct{}), make(chan struct{}) // the four in flight wait on first, every later send on then
|
||||
var arrived []string
|
||||
send := b.Send
|
||||
b.Send = func(ctx context.Context, p Publish) error {
|
||||
b.mu.Lock()
|
||||
arrived = append(arrived, p.Name)
|
||||
gate := map[bool]chan struct{}{true: first, false: then}[len(arrived) <= PublishSendsAtOnce]
|
||||
b.mu.Unlock()
|
||||
<-gate
|
||||
return send(ctx, p)
|
||||
}
|
||||
b.Go = func(f func()) { go f() }
|
||||
b.drainLater(ctx, "oci")
|
||||
synctest.Wait() // four start sends in flight, the pass holding o5 and o6
|
||||
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
|
||||
t.Fatalf("queued %v (%v)", q, err)
|
||||
}
|
||||
close(first)
|
||||
synctest.Wait() // the sends after the four: with the live one among them, or o5 and o6 alone when the pass goes on
|
||||
b.mu.Lock()
|
||||
next := slices.Clone(arrived[PublishSendsAtOnce:])
|
||||
b.mu.Unlock()
|
||||
if !slices.Contains(next, "live/prog") {
|
||||
t.Errorf("after the four in flight the sends are %v, want the live one among them, the pass listed again", next)
|
||||
}
|
||||
close(then)
|
||||
synctest.Wait()
|
||||
if len(arrived) != 7 {
|
||||
t.Errorf("sent %v, want all seven", arrived)
|
||||
}
|
||||
})
|
||||
})
|
||||
t.Run("a build a newer patch or release of its series supersedes is retired at the pass and at the start, never by a newer series", func(t *testing.T) {
|
||||
t.Run("a build a newer patch or release of its series supersedes is ended at the send, never by a newer series", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.queueAt(t, v, PublishClassStart, b.now)
|
||||
b.Drain(ctx, "oci")
|
||||
if len(b.sent) != 1 || len(b.ended) != 0 {
|
||||
t.Errorf("with a standing 0.2: sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", b.sent, b.ended, v.Version)
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); !f.Sent || len(b.sent) != 1 || len(b.ended) != 0 {
|
||||
t.Errorf("with a standing 0.2: %+v, sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", f, b.sent, b.ended, v.Version)
|
||||
}
|
||||
rel := &reserveMem{mem: newMem()}
|
||||
raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.3"}, {Fingerprint: "g", Version: "0.1.9"}}})
|
||||
_, _ = rel.Create(ctx, ReleasedKey("m"), raw)
|
||||
b.Reserved.Releases = rel
|
||||
b.sent = nil
|
||||
b.queueAt(t, v, PublishClassStart, b.now)
|
||||
b.Drain(ctx, "oci")
|
||||
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("at the pass: sent %v, ended %v, want m %s retired by the release 0.1.9", b.sent, b.ended, v.Version)
|
||||
}
|
||||
b.ended = nil
|
||||
b.queueAt(t, v, PublishClassStart, b.now)
|
||||
b.Go = func(func()) {}
|
||||
if err := b.Resume(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version)
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("at the send: %+v, sent %v, ended %v, want m %s retired by the release 0.1.9", f, b.sent, b.ended, v.Version)
|
||||
}
|
||||
only := &reserveMem{mem: newMem()}
|
||||
raw, _ = json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.0"}}})
|
||||
@@ -473,13 +359,27 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
|
||||
t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
b.queueAt(t, v, PublishClassStart, b.now)
|
||||
if _, err := b.Reserved.Reserve(ctx, "m", "0.1", []string{"m:prog"}, Source{Fingerprint: "src2:other"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.Drain(ctx, "oci")
|
||||
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("sent %v, ended %v: want nothing sent and the item ended", b.sent, b.ended)
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("%+v, sent %v, ended %v: want nothing sent and the item ended", f, b.sent, b.ended)
|
||||
}
|
||||
})
|
||||
t.Run("an item whose artifacts are published or refused as not stored ends unsent", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
b.refuse = "not stored: no blob"
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 {
|
||||
t.Errorf("a not stored refusal answered %+v, sent %d: want it ended after one send", f, len(b.sent))
|
||||
}
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 {
|
||||
t.Errorf("again: %+v, sent %d: want it ended unsent", f, len(b.sent))
|
||||
}
|
||||
b.refuse = "the holder is full"
|
||||
w := b.reserve(t, "n")
|
||||
if f := b.SendQueued(ctx, b.queueAt(w, PublishClassLive, b.now)); f.Failed == "" || f.Sent {
|
||||
t.Errorf("a refusal in other words answered %+v, want it failed, handed back", f)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -527,7 +427,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
t.Errorf("a state one page holds is answered capped: %+v", a.Read)
|
||||
}
|
||||
queue.All = func() (StatePage, error) { return StatePage{Values: vals, More: true, Keys: 5000, Bytes: 3 << 20}, nil }
|
||||
if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishQueueState ||
|
||||
if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishesLocal ||
|
||||
c.Read[0].KeysRead != 2000 || c.Read[0].StateKeys != 5000 || c.Read[0].StateBytes != 3<<20 || c.Total != 2000 {
|
||||
t.Errorf("a state past its one page answered capped %v, read %+v (%v): want capped, 2000 keys read of 5000, its size said (H-A-293)", c.Capped, c.Read, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user