B4: the publish items travel on the work queue publishes, live first

The bucket publish-queue listed and reread itself before each send; its items
now ride the builder's work queue publishes, pulled per holder and class by
consumers the builder makes as it learns a holder, the live class first, a
failed send handed back with the pause, an item acknowledged when its publish
is heard and terminated when its build is withdrawn, superseded or dropped.
builder_publish_drop and builder_publish_first act through reasks after the
hand-act log; builder_publish_queue reads the queue through the runtime. The
old state's items are moved once at the start, then dropped through the bus
module.

Builds: design 47 H2 §793, §820; H-A-290, H-A-291, H-A-440, H-A-446 (2), H-A-448
This commit is contained in:
2026-11-22 02:22:59 +01:00
parent b45f12f70f
commit d6d15a5497
16 changed files with 1318 additions and 553 deletions
@@ -433,8 +433,9 @@ func main() {
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
Newest: newestVia(stdio.Ask), Steps: r}))
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState)
tools = append(tools, republishTools(readerOf(qs), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
as := stdio.State(versions.ActedState)
tools = append(tools, publishActTools(buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
tools = append(tools, republishTools(workQueueReader(stdio.BusState, versions.PublishesLocal), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
ws := stdio.State(build.WaitsState)
wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) {
e, err := ws.Get(key)
@@ -4,11 +4,13 @@ import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
@@ -53,6 +55,95 @@ func onePage(page func(after string, maxBytes int, budget time.Duration) (stdio.
return out, nil
}
// workQueuePageItems and workQueueItemsRead bound the read of the work queue publishes: pages of 500 items, at most 4,000 read
// (H-A-293: past them the answer is capped, naming the items read and the queue's size).
const (
workQueuePageItems = 500
workQueueItemsRead = 4000
)
// workQueueReader is the builder's work queue local as the versions package's reader (design 47 H2 §793, H-A-448): its items read
// through the tool runtime's stream read in pages, each keyed by its sequence, and its size through the work queue's size; a
// read past workQueueItemsRead is capped. call is stdio.BusState.
func workQueueReader(call func(method string, params any) (json.RawMessage, error), local string) versions.Reader {
none := func() ([]string, error) { return nil, errors.ErrUnsupported }
return versions.Reader{Keys: none, All: func() (versions.StatePage, error) {
out := versions.StatePage{Values: map[string]json.RawMessage{}}
from := uint64(0)
for {
raw, err := call(buscalls.StreamRead, buscalls.StreamReadRequest{Stream: local, From: from, Limit: workQueuePageItems})
if err != nil {
return out, err
}
var pg buscalls.StreamPage
if err := json.Unmarshal(raw, &pg); err != nil {
return out, fmt.Errorf("the work queue %s answered what is no page of items: %w", local, err)
}
for _, m := range pg.Messages {
out.Values[strconv.FormatUint(m.Sequence, 10)] = m.Body
}
if pg.Next == 0 || len(pg.Messages) == 0 {
return out, nil
}
if len(out.Values) >= workQueueItemsRead {
out.More = true
break
}
from = pg.Next
}
raw, err := call(buscalls.WorkQueueSize, buscalls.WorkQueueRequest{Queue: local})
var size buscalls.WorkQueueSized
if err == nil {
err = json.Unmarshal(raw, &size)
}
if err != nil {
out.InfoWords = fmt.Sprintf("the work queue's size cannot be read: %v", err)
} else {
out.Keys, out.Bytes = size.Count, size.Bytes
}
return out, nil
}}
}
// publishActTools are builder_publish_drop and builder_publish_first (design 47 H2 §793): hand acts, recorded in the hand-act log
// first, then written to reasks for the builder, which owns the work queue, to perform and answer.
func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
tool := func(name, mode, desc, what string) stdio.Tool {
return stdio.Tool{Name: name, Description: desc,
Input: map[string]any{"type": "object", "required": []string{"module", "version", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the item's module"},
"version": map[string]any{"type": "string", "description": "the item's version, MAJOR.MINOR.PATCH"},
"why": map[string]any{"type": "string", "description": "why, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewPublishAct(mode, str(args, "module"), str(args, "version"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("%s %s %s by hand", what, str(args, "module"), str(args, "version")),
Why: str(args, "why"), Cause: name, Verb: mode}); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was done: %w", err)
}
raw, err := ask(key, value)
if err != nil {
return nil, err
}
a, err := build.DecodeReAsk(raw)
if err != nil {
return nil, err
}
if a.Refused {
return nil, errors.New(a.Answer)
}
return map[string]any{"key": key, "answer": a.Answer}, nil
}}
}
return []stdio.Tool{
tool(versions.PublishDropTool, build.ModePublishDrop, versions.PublishDropDescription, "dropped the publish of"),
tool(versions.PublishFirstTool, build.ModePublishFirst, versions.PublishFirstDescription, "put first the publish of"),
}
}
// republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act:
// it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer
// with the versions it queued.
@@ -3,10 +3,15 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
@@ -60,4 +65,58 @@ func TestAStateIsReadInOnePage(t *testing.T) {
t.Errorf("health four minutes in = %+v, want healthy", a)
}
})
// Design 47 H2 §793, H-A-448: builder_publish_queue reads the work queue publishes through the runtime's stream read, in pages,
// and a read past its bound is capped with the queue's size from the work queue's size; builder_publish_drop and
// builder_publish_first record the hand act first, then write their reasks value.
t.Run("the work queue is read in pages, capped past its bound", func(t *testing.T) {
for _, total := range []int{1200, 9000} {
var methods []string
call := func(method string, params any) (json.RawMessage, error) {
methods = append(methods, method)
if method == buscalls.WorkQueueSize {
return json.Marshal(buscalls.WorkQueueSized{Fill: buscalls.Fill{Count: uint64(total), Bytes: 4096}})
}
req := params.(buscalls.StreamReadRequest)
var pg buscalls.StreamPage
for n := req.From + 1; n <= uint64(total) && len(pg.Messages) < req.Limit; n++ {
pg.Messages = append(pg.Messages, buscalls.StreamMessage{Sequence: n, Body: json.RawMessage(`{"holder":"oci","class":"again"}`)})
pg.Next = n
}
if pg.Next == uint64(total) {
pg.Next = 0
}
return json.Marshal(pg)
}
pg, err := workQueueReader(call, versions.PublishesLocal).All()
capped := total > workQueueItemsRead
if err != nil || pg.More != capped || len(pg.Values) != min(total, workQueueItemsRead) || (capped && pg.Keys != uint64(total)) {
t.Errorf("%d items: read %d, more %v, keys %d (%v) after %v", total, len(pg.Values), pg.More, pg.Keys, err, methods)
}
}
})
t.Run("drop and first record the hand act, then ask the builder", func(t *testing.T) {
var order []string
ask := func(key string, value []byte) (json.RawMessage, error) {
order = append(order, "ask "+key)
r, _ := build.DecodeReAsk(value)
r.Answer = "done"
return json.Marshal(r)
}
acts := meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
order = append(order, "record")
return json.RawMessage(`{"id":"h1"}`), nil
}}
tools := publishActTools(ask, func() time.Time { return time.Unix(5, 0) }, acts)
for _, tool := range tools {
if _, err := tool.Run(map[string]any{"module": "m", "version": "0.1.2", "why": "a test"}); err != nil {
t.Fatalf("%s: %v", tool.Name, err)
}
}
if len(tools) != 2 || fmt.Sprint(order) != "[record ask publish-drop.5000000000 record ask publish-first.5000000000]" {
t.Errorf("tools %d, order %v: want each hand act recorded before its reasks value", len(tools), order)
}
if _, err := tools[0].Run(map[string]any{"module": "m", "why": "no version"}); err == nil {
t.Error("a drop naming no version is not refused")
}
})
}
@@ -8,6 +8,7 @@ import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
"io"
"io/fs"
"log"
@@ -39,6 +40,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/operatorask"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -268,10 +270,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.
acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider}
// A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G
// §588): its states publish-queue and acted, bound once the bus holds them.
queueKV := process.AwaitBucket(ctx, js, conn, own.queue, "no version is published again on an event or by builder_publish_again", logger.Printf)
// §588): its items on the work queue publishes its requires asks for, named by the bus's binding (design 47 H2 §793, H-A-448).
actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf)
republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}},
publishesBinding := process.NewBinding(meshbroker.Seat, cred.Module, calls.CallKept, logger.Printf, versions.PublishesLocal)
if err := publishesBinding.Register(hub); err != nil {
return fmt.Errorf("hearing the bus's binding of %s: %w", versions.PublishesLocal, err)
}
pq := &publishQueue{js: js, logf: logger.Printf,
binding: func() (membership.BusBinding, error) { return publishesBinding.Names(versions.PublishesLocal) }}
republish := &versions.Republisher{Reserved: reserved.r, Put: pq.put,
Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk),
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
@@ -300,6 +307,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
if err != nil {
return err
@@ -318,15 +326,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
},
markHeld: reserved.markHeld, moveElsewhere: reserved.movedElsewhere,
allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { return reserved.r.All(ctx) },
send: republish.SendQueued,
retired: reserved.r.Retired} // only each module's newest done build is asked at the start (H-A-291 (1))
// A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)).
republish.Ended = items.Done
republish.Ended = items.Ended
pq.take = items.take
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept)
acts.Republish = republish.Republish
acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) {
return pq.drop(ctx, items, module, version, by, why)
}
acts.PublishFirst = func(ctx context.Context, module, version, _, _ string) (string, error) {
return pq.first(ctx, items, module, version)
}
// builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does.
acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf)
again := &republishing{rp: republish}
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
decided := &decisions{subject: oc.SubjectDecided(cred.Module)}
retakeActed := awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}
@@ -340,8 +355,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return fmt.Errorf("hearing the operator's answers: %w", err)
}
retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}}
queueKV.Ready(again.bound(ctx, js, logger.Printf))
actedKV.Ready(again.bound(ctx, js, logger.Printf))
// Once the bus names the work queue: what waits on it is pulled, and the old state's items are put on it once (§793).
publishesBinding.Start()
go func() {
publishesBinding.Ask(versions.PublishesLocal)
if err := pq.learnAll(ctx); err != nil {
logger.Printf("the publishes waiting on the work queue %s are not pulled yet: %v; each holder's are pulled when an item of it is put", versions.PublishesLocal, err)
}
moveOld(ctx, js, own.queue, cred.Module, republish, calls.Call, logger.Printf)
}()
errs := make(chan error, 10)
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466)
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
@@ -0,0 +1,172 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"fmt"
"slices"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/membership"
"mesh-delivery-builder/internal/versions"
)
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder.
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and asks the
// bus module to drop the old state.
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
b := startBed(t)
ctx := b.ctx
bind := membership.BusBinding{Local: versions.PublishesLocal, Stream: "mesh-delivery-builder_publishes", Subject: "mesh.bus.mesh-delivery-builder.publishes"}
if _, err := b.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: bind.Stream, Subjects: []string{bind.Subject + ".>"},
Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil {
t.Fatal(err)
}
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
var mu sync.Mutex
var sent []string
published := map[string]bool{} // the modules whose publish is heard
sentCh := make(chan string, 200)
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5},
now: time.Now, logf: t.Logf,
reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) {
mu.Lock()
defer mu.Unlock()
return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil
},
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
mu.Lock()
sent = append(sent, q.Module)
mu.Unlock()
sentCh <- q.Module
return versions.SendFate{Sent: true}
}}
pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: func() (membership.BusBinding, error) { return bind, nil }}
item := func(module, class string) versions.QueuedPublish {
return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()}
}
away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled
data, _ := json.Marshal(q)
if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil {
t.Fatal(err)
}
}
for i := range 100 {
away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart))
}
away(item("dropme", versions.PublishClassStart))
away(item("live", versions.PublishClassLive))
if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil {
t.Fatalf("drop: %v", err)
} else {
t.Log(answer)
}
if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup {
t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err)
}
if err := pq.learnAll(ctx); err != nil {
t.Fatal(err)
}
select {
case first := <-sentCh:
if first != "live" {
t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first)
}
case <-time.After(30 * time.Second):
t.Fatal("nothing was sent")
}
for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard
select {
case <-sentCh:
case <-time.After(30 * time.Second):
t.Fatal("the again items were not sent")
}
}
deadline := time.After(60 * time.Second)
for { // each publish sent is heard: its item acknowledged, the next sent
mu.Lock()
n, done := len(sent), slices.Clone(sent)
for _, m := range done {
published[m] = true
}
mu.Unlock()
if n >= 101 {
break
}
for _, m := range done {
items.heard(ctx, m, "0.1.0")
}
select {
case <-sentCh:
case <-deadline:
t.Fatalf("sent %d of 101", n)
}
}
mu.Lock()
if slices.Contains(sent, "dropme") || len(sent) != 101 {
t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme"))
}
mu.Unlock()
// The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked).
old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)})
if err != nil {
t.Fatal(err)
}
call := func(string, any) (json.RawMessage, error) {
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
}
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf)
rv.filled.Store(true)
for _, m := range []string{"m1", "m2", "m3"} {
v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m})
if err != nil {
t.Fatal(err)
}
if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"})
if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil {
t.Fatal(err)
}
}
rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) {
data, _ := json.Marshal(q)
ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
if err != nil {
return false, err
}
return ack.Duplicate, nil
}, Started: time.Now(), Logf: t.Logf}
var drops []string
drop := func(tool string, body any) (json.RawMessage, error) {
drops = append(drops, fmt.Sprint(tool, body))
return json.RawMessage(`{}`), nil
}
for range 2 {
moveOld(ctx, b.js, builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState), "mesh-delivery-builder", rp, drop, t.Logf)
}
st, err := b.js.Stream(ctx, bind.Stream)
if err != nil {
t.Fatal(err)
}
info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>"))
if err != nil {
t.Fatal(err)
}
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 || len(drops) != 2 {
t.Errorf("the move run twice left %v on the work queue, dropped %v: want each of the three once, as again, and the old state dropped through the bus module", info.State.Subjects, drops)
}
}
@@ -7,20 +7,25 @@ import (
"fmt"
"io/fs"
"os"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/membership"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588): each reservation of a
// done build is a publish item in the builder's re-ask state, the mechanism builder_reserve_done and publish again use. It is
// asked when the build ends and at the builder's start; each try queues the publish and writes the item again with one more try
// and a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue.
// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588; H2 §793): each
// reservation of a done build is a publish item in the builder's re-ask state, asked when the build ends and at the builder's
// start; its try puts the publish on the work queue publishes, and each send counts one try in the item and hands a failed one
// back with a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue.
// publishSettings are the settings publish-pause, publish-pause-max and publish-tries.
type publishSettings struct {
@@ -28,8 +33,8 @@ type publishSettings struct {
Tries int
}
// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 8 tries.
var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 8}
// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 5 tries (H-A-446 (2)).
var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 5}
// pauseAfter is the pause after the try-th try: the setting doubled per try, up to the maximum.
func (s publishSettings) pauseAfter(try int) time.Duration {
@@ -61,10 +66,15 @@ type publishItems struct {
// publish there (H-A-275). Nil does neither.
markHeld func(ctx context.Context, v versions.ReservedVersion, held []versions.HeldArtifact)
moveElsewhere func(ctx context.Context, v versions.ReservedVersion, j versions.Judged) error
set publishSettings
now func() time.Time
after func(time.Duration) <-chan time.Time
logf func(string, ...any)
// send sends one item pulled from the work queue (Republisher.SendQueued); nil sends none.
send func(ctx context.Context, q versions.QueuedPublish) versions.SendFate
set publishSettings
now func() time.Time
after func(time.Duration) <-chan time.Time
logf func(string, ...any)
mu sync.Mutex
held map[string][]heldItem // "<module> <version>": the items sent, their publish not yet heard
}
func (p *publishItems) wait(ctx context.Context, d time.Duration) bool {
@@ -130,8 +140,11 @@ func (p *publishItems) AskAllDone(ctx context.Context) {
}
}
// Done deletes a version's publish item: its release is written.
// Done deletes a version's publish item and acknowledges its held items: its release is written.
func (p *publishItems) Done(ctx context.Context, module, version string) {
for _, h := range p.settle(module, version, func(heldItem) bool { return true }) {
_ = h.m.Ack()
}
kv := p.reasks()
if kv == nil {
return
@@ -141,9 +154,10 @@ func (p *publishItems) Done(ctx context.Context, module, version string) {
}
}
// Try waits for the item's next try, then, while the version is still reserved, queues its publish and writes the item again with
// one more try and the pause after it; once the tries are spent it ends the item loudly. A version no longer reserved (released,
// dropped or taken over) has its item deleted; one whose artifacts are published keeps it until its release is written.
// Try waits for the item's next try, then, while the version is still reserved, puts its publish on the work queue publishes
// under its message id (design 47 H2 §793; H-A-448): an item the queue holds already is put once. It counts no try: each send
// does, and hands its item back with the pause (take). A put that queued nothing is put again after the pause. A version no
// longer reserved (released, dropped or taken over) has its item deleted.
func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk) {
if !p.wait(ctx, r.NextTry.Sub(p.now())) {
return
@@ -157,37 +171,215 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre
p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err)
return
}
if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn: never because its artifacts are published (A-456b, H-A-290 (2))
if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn (A-456b, H-A-290 (2))
p.logf("%s %s: the publish item is ended: %s", r.Module, r.Version, why)
_ = kv.Delete(ctx, e.Key())
return
}
queued, err := p.republish(ctx, r.Module, r.Version)
if err == nil && len(queued) > 0 {
return
}
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
p.logf("%s %s: the publish is not on the work queue %s (%v); it is put again after %s", r.Module, r.Version, versions.PublishesLocal, err, p.set.pauseAfter(r.Tries))
if data, err := json.Marshal(r); err == nil {
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.logf("%s: the publish item could not be written: %v", e.Key(), err)
}
}
}
// itemMsg is what take reads of an item pulled from the work queue (jetstream.Msg).
type itemMsg interface {
Data() []byte
Ack() error
NakWithDelay(time.Duration) error
Term() error
}
// take handles one item pulled from the work queue (§793; H-A-440 (2), H-A-448): an item whose publish item ended (dropped, or
// its tries spent) or whose version never publishes is terminated; one whose artifacts are all published is acknowledged; once
// publish-tries sends are spent it is terminated and version-unreleased said; while a published of it may be unheard it is handed
// back with the pause, and one the registries' record confirms is acknowledged (H-A-274); else it is sent, the try counted in its
// publish item, and a send that fails handed back with the pause as its delay, so the queue delivers it again and nothing is put
// again. A sent item is acknowledged when its publish is heard (heard, Done) and delivered again after the ack wait, unheard.
func (p *publishItems) take(ctx context.Context, m itemMsg) {
var q versions.QueuedPublish
if err := json.Unmarshal(m.Data(), &q); err != nil || q.Module == "" || q.Version == "" {
p.logf("an item of the work queue %s that is no publish is terminated: %v", versions.PublishesLocal, err)
_ = m.Term()
return
}
kv := p.reasks()
if kv == nil {
p.logf("%s %s: the re-ask state is not on the bus yet; the item is handed back", q.Module, q.Version)
_ = m.NakWithDelay(p.set.Pause)
return
}
key := versions.PublishItemKey(q.Module, q.Version)
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) { // put by builder_publish_again or the move: its tries are counted from now
if err = p.Ask(ctx, q.Module, q.Version); err == nil {
e, err = kv.Get(ctx, key)
}
}
if err != nil {
p.logf("%s %s: its publish item cannot be read (%v); the item is handed back", q.Module, q.Version, err)
_ = m.NakWithDelay(p.set.Pause)
return
}
r, err := build.DecodeReAsk(e.Value())
if err != nil {
_ = m.Term()
return
}
if r.Answer != "" {
p.logf("%s %s at %s is terminated: its publish item ended: %s", q.Module, q.Version, q.Holder, r.Answer)
_ = m.Term()
return
}
v, found, err := p.reserved(ctx, q.Module, q.Version)
if err != nil {
p.logf("%s %s: its reservation cannot be read (%v); the item is handed back", q.Module, q.Version, err)
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
return
}
if why := versions.NotStanding(v, found); why != "" {
p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, why)
_ = m.Term()
_ = kv.Delete(ctx, key)
return
}
if allPublished(v, q) {
p.logf("%s %s at %s is acknowledged: its artifacts are published", q.Module, q.Version, q.Holder)
_ = m.Ack()
return
}
if r.Tries >= p.set.Tries {
_ = m.Term()
r.Answer = fmt.Sprintf("the publish of %s %s was tried %d times and the version is not released; builder_publish_again tries it again", r.Module, r.Version, r.Tries)
r.Answered = p.now().UTC()
p.logf("%s: %s", e.Key(), r.Answer)
p.logf("%s: %s", key, r.Answer)
if p.endedWith != nil {
p.endedWith(ctx, v, "the publish ended after "+fmt.Sprint(r.Tries)+" tries")
}
} else if why, gone := p.notAgain(ctx, v); gone {
return // moved to another number, whose own item is asked (H-A-275)
} else if why != "" {
// A published of the version may be unheard, or the registry's record ended it: no ask, and no try spent (H-A-272, H-A-274).
p.logf("%s %s: the publish is not asked again: %s; tried again after %s", r.Module, r.Version, why, p.set.pauseAfter(r.Tries))
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
} else {
r.Tries++
if _, err := p.republish(ctx, r.Module, r.Version); err != nil {
p.logf("%s %s: try %d queued no publish: %v", r.Module, r.Version, r.Tries, err)
}
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
p.write(ctx, kv, key, r, e.Revision())
return
}
if why, gone := p.notAgain(ctx, v); gone {
_ = m.Term() // moved to another number, whose own item is asked (H-A-275)
return
} else if why != "" {
if v2, ok, err := p.reserved(ctx, q.Module, q.Version); err == nil && ok && allPublished(v2, q) {
p.logf("%s %s at %s is acknowledged: %s", q.Module, q.Version, q.Holder, why)
_ = m.Ack()
return
}
p.logf("%s %s: the publish is not sent again: %s; handed back for %s", q.Module, q.Version, why, p.set.pauseAfter(r.Tries))
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
return
}
fate := versions.SendFate{Failed: "this builder sends no publish"}
if p.send != nil {
fate = p.send(ctx, q)
}
if fate.Ended != "" {
p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, fate.Ended)
_ = m.Term()
return
}
r.Tries++
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
p.write(ctx, kv, key, r, e.Revision())
if fate.Failed != "" {
p.logf("%s %s at %s: try %d sent no publish: %s; handed back for %s", q.Module, q.Version, q.Holder, r.Tries, fate.Failed, p.set.pauseAfter(r.Tries))
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
return
}
p.hold(q, m)
}
// write writes a publish item at its revision; a newer write wins.
func (p *publishItems) write(ctx context.Context, kv jetstream.KeyValue, key string, r build.ReAsk, rev uint64) {
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.logf("%s: the publish item could not be written: %v", e.Key(), err)
if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.logf("%s: the publish item could not be written: %v", key, err)
}
}
// allPublished says none of q's artifacts is missing from v: its publish is heard.
func allPublished(v versions.ReservedVersion, q versions.QueuedPublish) bool {
missing := map[string]bool{}
for _, ref := range v.Missing() {
missing[ref] = true
}
for ref := range q.Artifacts {
if missing[ref] {
return false
}
}
return true
}
// heldItem is an item sent and not yet acknowledged: its publish is not heard yet.
type heldItem struct {
q versions.QueuedPublish
m itemMsg
}
// hold keeps a sent item until its publish is heard.
func (p *publishItems) hold(q versions.QueuedPublish, m itemMsg) {
p.mu.Lock()
defer p.mu.Unlock()
if p.held == nil {
p.held = map[string][]heldItem{}
}
k := q.Module + " " + q.Version
p.held[k] = append(p.held[k], heldItem{q: q, m: m})
}
// settle takes every held item of module's version that pick picks out of the held ones.
func (p *publishItems) settle(module, version string, pick func(heldItem) bool) []heldItem {
p.mu.Lock()
defer p.mu.Unlock()
k := module + " " + version
var out, keep []heldItem
for _, h := range p.held[k] {
if pick(h) {
out = append(out, h)
} else {
keep = append(keep, h)
}
}
if len(keep) == 0 {
delete(p.held, k)
} else {
p.held[k] = keep
}
return out
}
// heard acknowledges each held item of module's version whose artifacts are all published now: its publish is heard (§793).
func (p *publishItems) heard(ctx context.Context, module, version string) {
v, found, err := p.reserved(ctx, module, version)
if err != nil || !found {
return
}
for _, h := range p.settle(module, version, func(h heldItem) bool { return allPublished(v, h.q) }) {
p.logf("%s %s at %s is acknowledged: its publish is heard", module, version, h.q.Holder)
_ = h.m.Ack()
}
}
// Ended deletes a version's publish item and terminates its held items: its build is withdrawn or superseded (H-A-290 (2),
// H-A-291 (1)); an item still on the work queue is terminated when it is pulled.
func (p *publishItems) Ended(ctx context.Context, module, version string) {
p.Done(ctx, module, version)
for _, h := range p.settle(module, version, func(heldItem) bool { return true }) {
_ = h.m.Term()
}
}
@@ -312,10 +504,11 @@ func publishSettingsOf(getenv func(string) string) (publishSettings, error) {
return publishSettings{Pause: time.Duration(pause) * time.Second, PauseMax: time.Duration(pmax) * time.Minute, Tries: tries}, nil
}
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b): a done build asks its item, a
// release, a drop and a takeover end it, and a release is recorded in releases, the last kept of each module.
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b, H2 §793): a done build asks its
// item, a release ends it and acknowledges its held items, a drop and a takeover end it and terminate them, a published heard
// acknowledges those it publishes, and a release is recorded in releases, the last kept of each module.
func wirePublishItems(rv *reservations, items *publishItems, releases versions.ReserveStore, kept int) {
rv.built, rv.released = items.Ask, items.Done
rv.built, rv.released, rv.itemEnded, rv.itemHeard = items.Ask, items.Done, items.Ended, items.heard
rv.r.ReleasedKept, rv.r.Releases = kept, releases
}
@@ -330,3 +523,366 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
}
return out
}
// publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-448): asked for in
// its requires and named by the bus's binding; each item on `<binding subject>.publish.<holder>.<class>.<module>.<version>` under
// the message id `<holder>.<class>.<module>.<version>`. The builder makes the pull consumers at run time as it learns a holder,
// `publishes-<holder>-<class>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live
// class first, at most four sends at once per holder.
type publishQueue struct {
js jetstream.JetStream
binding func() (membership.BusBinding, error)
ackWait time.Duration
take func(ctx context.Context, m itemMsg)
logf func(string, ...any)
mu sync.Mutex
pulling map[string]bool
}
// The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's
// largest; the builder ends an item at publish-tries anyway.
const (
publishesMaxAckPending = versions.PublishSendsAtOnce
publishesMaxDeliver = 10
)
// publishesConsumer is the durable consumer of holder's items of class.
func publishesConsumer(holder, class string) string {
return versions.PublishesLocal + "-" + holder + "-" + class
}
// subject is the item's subject on the bus.
func (pq *publishQueue) subject(b membership.BusBinding, q versions.QueuedPublish) string {
return b.Subject + "." + versions.PublishEvent(q.Holder, q.Class, q.Module, q.Version)
}
// put puts q on the work queue under its message id; duplicate says the queue holds that id within its window. A put the full
// queue refuses is versions.ErrQueueFull. The item's holder is learnt: its pull starts.
func (pq *publishQueue) put(ctx context.Context, q versions.QueuedPublish) (bool, error) {
b, err := pq.binding()
if err != nil {
return false, err
}
data, err := json.Marshal(q)
if err != nil {
return false, err
}
ack, err := pq.js.PublishMsg(ctx, &nats.Msg{Subject: pq.subject(b, q), Data: data}, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
if err != nil {
if w := err.Error(); strings.Contains(w, "maximum bytes") || strings.Contains(w, "maximum messages") || errors.Is(err, jetstream.ErrMaxBytesExceeded) {
return false, fmt.Errorf("%w: %v", versions.ErrQueueFull, err)
}
return false, err
}
pq.learn(ctx, q.Holder)
return ack.Duplicate, nil
}
// learnAll learns every holder whose items the work queue holds: at the start, so what waits there is pulled.
func (pq *publishQueue) learnAll(ctx context.Context) error {
b, err := pq.binding()
if err != nil {
return err
}
st, err := pq.js.Stream(ctx, b.Stream)
if err != nil {
return err
}
info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.>"))
if err != nil {
return err
}
for subj := range info.State.Subjects {
if holder, _, ok := strings.Cut(strings.TrimPrefix(subj, b.Subject+".publish."), "."); ok {
pq.learn(ctx, holder) // a holder is a module, its name one token
}
}
return nil
}
// learn starts holder's pull once.
func (pq *publishQueue) learn(ctx context.Context, holder string) {
pq.mu.Lock()
if pq.pulling == nil {
pq.pulling = map[string]bool{}
}
if pq.pulling[holder] {
pq.mu.Unlock()
return
}
pq.pulling[holder] = true
pq.mu.Unlock()
if err := pq.pull(ctx, holder); err != nil {
pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err)
pq.mu.Lock()
delete(pq.pulling, holder)
pq.mu.Unlock()
}
}
// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item
// waits while a live one waits on the queue.
func (pq *publishQueue) pull(ctx context.Context, holder string) error {
b, err := pq.binding()
if err != nil {
return err
}
chans := map[string]chan jetstream.Msg{}
var live jetstream.Consumer
for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { // the again first: the live is first by the dispatch, not by luck
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class),
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy,
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
if err != nil {
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err)
}
ch := make(chan jetstream.Msg, publishesMaxAckPending)
cc, err := c.Consume(func(m jetstream.Msg) { ch <- m }, jetstream.PullMaxMessages(publishesMaxAckPending))
if err != nil {
return fmt.Errorf("the consumer %s is not pulled: %w", publishesConsumer(holder, class), err)
}
go func() { <-ctx.Done(); cc.Stop() }()
chans[class] = ch
if class == versions.PublishClassLive {
live = c
}
}
liveWaits := func() bool {
info, err := live.Info(ctx)
return err == nil && info.NumPending > 0
}
pq.logf("pulling the publishes of %s, the live first", holder)
go pq.dispatch(ctx, chans[versions.PublishClassLive], chans[versions.PublishClassStart], liveWaits)
return nil
}
// dispatch runs the items of one holder, at most four at once, every live item before an again one: an again item pulled while
// a live one waits on the queue (liveWaits) runs after it.
func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstream.Msg, liveWaits func() bool) {
sem := make(chan struct{}, versions.PublishSendsAtOnce)
run := func(m jetstream.Msg) {
go func() {
defer func() { <-sem }()
pq.take(ctx, m)
}()
}
var spare jetstream.Msg // an again item pulled while a live one waited
for {
select {
case sem <- struct{}{}:
case <-ctx.Done():
return
}
select {
case m := <-live:
run(m)
continue
default:
}
m := spare
spare = nil
if m == nil {
select {
case lm := <-live:
run(lm)
continue
case m = <-again:
case <-ctx.Done():
return
}
}
if liveWaits() {
spare = m
select {
case lm := <-live:
run(lm)
case <-ctx.Done():
return
}
continue
}
select {
case lm := <-live: // came while the queue was asked
spare = m
run(lm)
default:
run(m)
}
}
}
// itemsOf are the items the work queue holds of module's version in class (empty for any class), each with its subject.
func (pq *publishQueue) itemsOf(ctx context.Context, module, version, class string) (map[string]versions.QueuedPublish, jetstream.Stream, error) {
b, err := pq.binding()
if err != nil {
return nil, nil, err
}
st, err := pq.js.Stream(ctx, b.Stream)
if err != nil {
return nil, nil, err
}
cls := "*"
if class != "" {
cls = class
}
info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.*."+cls+"."+versions.EscapeToken(module)+"."+versions.EscapeToken(version)))
if err != nil {
return nil, nil, err
}
out := map[string]versions.QueuedPublish{}
for subj := range info.State.Subjects {
raw, err := st.GetLastMsgForSubject(ctx, subj)
if err != nil {
continue
}
var q versions.QueuedPublish
if json.Unmarshal(raw.Data, &q) == nil {
out[subj] = q
}
}
return out, st, nil
}
// drop is builder_publish_drop (§793): the version's publish item is ended with who dropped it and why, so no item of it is ever
// sent or put again; its items waiting on the work queue are removed, and one held is terminated.
func (pq *publishQueue) drop(ctx context.Context, items *publishItems, module, version, by, why string) (string, error) {
qs, st, err := pq.itemsOf(ctx, module, version, "")
if err != nil {
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
}
if err := items.endItem(ctx, module, version, fmt.Sprintf("dropped by %s: %s", by, why)); err != nil {
return "", err
}
for subj := range qs {
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
return "", fmt.Errorf("the item %s is not terminated: %w", subj, err)
}
}
for _, h := range items.settle(module, version, func(heldItem) bool { return true }) {
_ = h.m.Term()
}
return fmt.Sprintf("the publish of %s %s is dropped: %d items terminated, and none is sent or put again", module, version, len(qs)), nil
}
// first is builder_publish_first (§793): each again item of the version is put again as live, under the message id
// `<holder>.live.<module>.<version>`, and the old terminated; a live id the queue holds within its window takes nothing, and the
// old stays, said.
func (pq *publishQueue) first(ctx context.Context, items *publishItems, module, version string) (string, error) {
qs, st, err := pq.itemsOf(ctx, module, version, versions.PublishClassStart)
if err != nil {
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
}
if len(qs) == 0 {
return "", fmt.Errorf("the work queue %s holds no again item of %s %s", versions.PublishesLocal, module, version)
}
var moved, kept []string
for subj, q := range qs {
q.Class = versions.PublishClassLive
dup, err := pq.put(ctx, q)
if err != nil {
return "", fmt.Errorf("%s %s is not put again as live at %s: %w", module, version, q.Holder, err)
}
if dup {
kept = append(kept, q.Holder)
continue
}
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
return "", fmt.Errorf("the again item of %s %s at %s is not terminated: %w", module, version, q.Holder, err)
}
moved = append(moved, q.Holder)
}
for _, h := range items.settle(module, version, func(h heldItem) bool { return h.q.Class == versions.PublishClassStart }) {
_ = h.m.Term()
}
sort.Strings(moved)
answer := fmt.Sprintf("%s %s is put first as live at %s, its again item terminated", module, version, strings.Join(moved, ", "))
if len(kept) > 0 {
sort.Strings(kept)
answer += fmt.Sprintf("; at %s the work queue holds its live id within the duplicates window, so its again item stays", strings.Join(kept, ", "))
}
return answer, nil
}
// endItem writes the version's publish item ended with answer, made when none stands: Try and take then leave it, and a start
// asks it no more.
func (p *publishItems) endItem(ctx context.Context, module, version, answer string) error {
kv := p.reasks()
if kv == nil {
return errors.New("the re-ask state is not on the bus yet")
}
key := versions.PublishItemKey(module, version)
for range 5 {
r := build.ReAsk{V: 1, Why: "the build ended with its version unreleased", By: "the builder", At: p.now().UTC(), Mode: build.ModePublish, Module: module, Version: version}
e, err := kv.Get(ctx, key)
var rev uint64
switch {
case errors.Is(err, jetstream.ErrKeyNotFound):
case err != nil:
return err
default:
if r, err = build.DecodeReAsk(e.Value()); err != nil {
return err
}
rev = e.Revision()
}
r.Answer, r.Answered = answer, p.now().UTC()
data, err := json.Marshal(r)
if err != nil {
return err
}
if rev == 0 {
_, err = kv.Create(ctx, key, data)
} else {
_, err = kv.Update(ctx, key, data, rev)
}
if err == nil {
return nil
}
if !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return err
}
}
return errors.New("the publish item of " + module + " " + version + " changed five times while it was ended")
}
// moveOld puts the items of the old state publish-queue on the work queue once at the start (§793): each under its message id, so
// a move run twice puts each once within the duplicates window; then the bucket is dropped through the bus module (its tool
// nats_drop, on the operator's warrant). A builder whose bus holds no such bucket moves nothing.
func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string, rp *versions.Republisher, call toolCaller, logf func(string, ...any)) {
kv, err := js.KeyValue(ctx, bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
return
}
if err != nil {
logf("the old state %s cannot be read, so its items are not moved to the work queue %s: %v", versions.PublishQueueState, versions.PublishesLocal, err)
return
}
keys, err := kv.ListKeys(ctx)
if err != nil {
logf("the old state %s cannot be listed, so its items are not moved: %v", versions.PublishQueueState, err)
return
}
var vals []json.RawMessage
for k := range keys.Keys() {
if e, err := kv.Get(ctx, k); err == nil {
vals = append(vals, json.RawMessage(e.Value()))
}
}
n, err := rp.MoveQueued(ctx, vals)
if err != nil {
logf("the old state %s is not dropped: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
return
}
logf("the old state %s: %d of its %d items put on the work queue %s; it is dropped through the bus module", versions.PublishQueueState, n, len(vals), versions.PublishesLocal)
if call == nil {
return
}
if _, err := call(natsDropTool, map[string]any{"module": module, "local": versions.PublishQueueState,
"why": "its items are on the builder's work queue " + versions.PublishesLocal + " (design 47 H2 section 793)"}); err != nil {
logf("the old state %s is not dropped through the bus module: %v", versions.PublishQueueState, err)
}
}
// natsDropTool is the bus module's drop of an entry no accepted manifest asks for (H-A-126), which the old state goes by.
const natsDropTool = "nats.nats_drop"
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"path/filepath"
@@ -89,6 +90,29 @@ func (k *itemKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOp
return nil
}
// itemMsgFake is an item pulled from the work queue: what take settled it with.
type itemMsgFake struct {
data []byte
settled []string // ack, term, nak <delay>
}
func (m *itemMsgFake) Data() []byte { return m.data }
func (m *itemMsgFake) Ack() error { m.settled = append(m.settled, "ack"); return nil }
func (m *itemMsgFake) Term() error { m.settled = append(m.settled, "term"); return nil }
func (m *itemMsgFake) NakWithDelay(d time.Duration) error {
m.settled = append(m.settled, "nak "+d.String())
return nil
}
func queuedMsg(t *testing.T, module, version string) *itemMsgFake {
t.Helper()
raw, err := json.Marshal(versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: version, Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: versions.PublishClassLive})
if err != nil {
t.Fatal(err)
}
return &itemMsgFake{data: raw}
}
func decodeItem(t *testing.T, raw []byte) build.ReAsk {
t.Helper()
r, err := build.DecodeReAsk(raw)
@@ -98,21 +122,21 @@ func decodeItem(t *testing.T, raw []byte) build.ReAsk {
return r
}
// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a — a version's publish item is asked once, each try queues
// the publish and writes the item again with one more try and the pause doubled up to its maximum, the release deletes it, and
// after the setting's tries it ends with its words, raises the condition and stays; a version no longer reserved has it deleted.
// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a, H2 §793 (H-A-440 (2), H-A-448) — a version's publish
// item is asked once; its try puts the publish on the work queue and counts no try; each send counts one and a failed send hands
// the item back with the pause doubled up to its maximum; after the setting's tries the item is terminated and ends with its words,
// raises the condition and stays; an item whose artifacts are published is acknowledged; the release deletes it; a version no
// longer reserved has it deleted.
func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
ctx := context.Background()
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
now := time.Date(2026, 11, 17, 1, 0, 0, 0, time.UTC)
var waited []time.Duration
var queued, ended []string
var queued, ended, sent []string
reserved := true
v := versions.ReservedVersion{Module: "m", Version: "0.1.2", Artifacts: map[string]string{"m:a": "sha256:a"}, BuiltAt: now}
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: 30 * time.Second, PauseMax: 2 * time.Minute, Tries: 4},
now: func() time.Time { return now }, logf: t.Logf,
after: func(d time.Duration) <-chan time.Time {
waited = append(waited, d)
now = now.Add(d)
c := make(chan time.Time, 1)
c <- now
@@ -126,7 +150,11 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
},
republish: func(_ context.Context, m, ver string) ([]string, error) {
queued = append(queued, m+" "+ver)
return nil, errors.New("no holder")
return []string{m + " " + ver}, nil
},
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
sent = append(sent, q.Module+" "+q.Version)
return versions.SendFate{Failed: "no holder answers"}
},
endedWith: func(_ context.Context, v versions.ReservedVersion, why string) {
ended = append(ended, v.Version+": "+why)
@@ -135,25 +163,34 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
if err := p.Ask(ctx, "m", "0.1.2"); err != nil || p.Ask(ctx, "m", "0.1.2") != nil || len(kv.vals) != 1 {
t.Fatalf("asked: %v, %d items, want one item however often it is asked", err, len(kv.vals))
}
for i := 0; i < 5; i++ {
e, _ := kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
e, _ := kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
if len(queued) != 1 || decodeItem(t, kv.vals[key]).Tries != 0 {
t.Fatalf("the try queued %v, tries %d: want the publish put once, no try counted", queued, decodeItem(t, kv.vals[key]).Tries)
}
var settled []string
for range 5 {
m := queuedMsg(t, "m", "0.1.2")
p.take(ctx, m)
settled = append(settled, m.settled...)
}
got := decodeItem(t, kv.vals[key])
if len(queued) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") {
t.Fatalf("queued %v, item %+v, ended %v: want four tries, then the end said with its words and the condition", queued, got, ended)
if len(sent) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") {
t.Fatalf("sent %v, item %+v, ended %v: want four sends, then the end said with its words and the condition", sent, got, ended)
}
if want := []time.Duration{30 * time.Second, time.Minute, 2 * time.Minute, 2 * time.Minute}; len(waited) != len(want) || waited[0] != want[0] || waited[1] != want[1] || waited[2] != want[2] || waited[3] != want[3] {
t.Errorf("pauses before the tries and the end = %v, want %v (doubled, capped)", waited, want)
if want := "[nak 30s nak 1m0s nak 2m0s nak 2m0s term]"; fmt.Sprint(settled) != want {
t.Errorf("the items were settled %v, want %s (handed back with the pause doubled and capped, then terminated)", settled, want)
}
// An item whose artifacts are all published stays until its release is written, and is tried (A-456b).
queued = nil
m := queuedMsg(t, "m", "0.1.2")
if p.take(ctx, m); fmt.Sprint(m.settled) != "[term]" || len(sent) != 4 {
t.Errorf("an item of an ended publish item was settled %v, sent %d: want it terminated unsent", m.settled, len(sent))
}
// An item whose artifacts are all published is acknowledged unsent; its publish item stays until its release is written (A-456b).
_ = p.Ask(ctx, "m", "0.3.0")
pk := versions.PublishItemKey("m", "0.3.0")
pe, _ := kv.Get(ctx, pk)
p.Try(ctx, kv, pe, decodeItem(t, pe.Value()))
if _, ok := kv.vals[pk]; !ok || len(queued) != 1 || decodeItem(t, kv.vals[pk]).Tries != 1 {
t.Errorf("an item whose artifacts are published was deleted or not tried: %v, queued %v", kv.vals, queued)
m = queuedMsg(t, "m", "0.3.0")
if p.take(ctx, m); fmt.Sprint(m.settled) != "[ack]" || len(sent) != 4 || !kv.has(pk) {
t.Errorf("an item whose artifacts are published was settled %v, sent %d, kept %v: want it acknowledged unsent, its publish item kept", m.settled, len(sent), kv.has(pk))
}
kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{}
// At the start every reservation of a done build with no release is asked, one item each.
@@ -194,7 +231,7 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
queued = nil
_ = p.Ask(ctx, "m", "0.1.2")
v.Withdrawn = map[string]string{"0.1.2": "fp"}
e, _ := kv.Get(ctx, key)
e, _ = kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
if len(kv.vals) != 0 || len(queued) != 0 {
t.Errorf("a withdrawn reservation left its item %v, queued %v", kv.vals, queued)
@@ -209,7 +246,8 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
}
}
// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a — the three settings, 30 s, 30 min and 8 tries unless set, and a
// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a, H2 H-A-446 (2) — the three settings, 30 s, 30 min and 5 tries
// unless set, and a
// content that is no number is refused naming its file.
func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
dir := t.TempDir()
@@ -222,7 +260,7 @@ func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
}
env := map[string]string{}
got, err := publishSettingsOf(func(k string) string { return env[k] })
if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 8 {
if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 5 {
t.Fatalf("defaults = %+v (%v)", got, err)
}
env["MESH_BUILDER_PUBLISH_PAUSE_FILE"], env["MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE"], env["MESH_BUILDER_PUBLISH_TRIES_FILE"] = write("p", "5\n"), write("m", "2"), write("t", "3")
@@ -353,9 +391,12 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
asked, judged, marked, moved := 0, 0, 0, 0
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: defaultPublishSettings, now: func() time.Time { return now }, logf: t.Logf,
reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil },
republish: func(context.Context, string, string) ([]string, error) { asked++; return nil, nil },
unheard: func(context.Context) (int, error) { return c.unheard, c.countErr },
reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil },
send: func(context.Context, versions.QueuedPublish) versions.SendFate {
asked++
return versions.SendFate{Sent: true}
},
unheard: func(context.Context) (int, error) { return c.unheard, c.countErr },
judge: func(context.Context, versions.ReservedVersion) (versions.Judged, error) {
judged++
return c.judged, nil
@@ -374,20 +415,24 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
t.Fatal(err)
}
key := versions.PublishItemKey(v.Module, v.Version)
e, _ := kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
m := queuedMsg(t, v.Module, v.Version)
m.data = []byte(strings.ReplaceAll(string(m.data), `"node-tools:a"`, `"node-tools:program"`))
p.take(ctx, m)
if (asked == 1) != c.wantAsked || (marked == 1) != c.wantMark || (moved == 1) != c.wantMov {
t.Fatalf("asked %d, marked %d, moved %d; want asked %v, marked %v, moved %v", asked, marked, moved, c.wantAsked, c.wantMark, c.wantMov)
}
if c.wantMov {
if _, ok := kv.vals[key]; ok {
t.Error("the moved version's old item is written again")
if _, ok := kv.vals[key]; ok || fmt.Sprint(m.settled) != "[term]" {
t.Errorf("the moved version's old item is written again, or its item settled %v, not terminated", m.settled)
}
return
}
got := decodeItem(t, kv.vals[key])
if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || !got.NextTry.After(now) {
t.Errorf("tries %d, next try %s: want a try spent only when asked, and the item written again with its pause", got.Tries, got.NextTry)
if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || (c.wantAsked && !got.NextTry.After(now)) {
t.Errorf("tries %d, next try %s: want a try spent only when sent, the item written with its pause", got.Tries, got.NextTry)
}
if !c.wantAsked && fmt.Sprint(m.settled) != "[nak 30s]" {
t.Errorf("an item not sent was settled %v, want it handed back with the pause and no try spent", m.settled)
}
if c.unheard >= versions.UnheardBound && judged != 0 {
t.Error("the registry is read while a published may be unheard")
@@ -65,7 +65,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi
released(ctx, b, kv, e, r, logf)
return
}
if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
go versionActed(ctx, acts, kv, e, r, logf)
return
@@ -208,6 +208,16 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue
r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them"
}
}
} else if r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst {
act := acts.PublishDrop
if r.Mode == build.ModePublishFirst {
act = acts.PublishFirst
}
if act == nil {
err = errors.New("this builder keeps no work queue of publishes")
} else {
r.Answer, err = act(ctx, r.Module, r.Version, r.By, r.Why)
}
} else if r.Mode == build.ModeReserveDone {
if acts.ReserveDone == nil {
err = errors.New("this builder keeps no reservations")
@@ -6,11 +6,8 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -21,21 +18,6 @@ import (
// The builder's publish again (design 47 G §588, A-73, A-76): the wiring of versions.Republisher to the bus.
// awaitedQueue is the state publish-queue as versions.QueueStore: bound once the bus holds it; its bytes are the cap's.
type awaitedQueue struct{ awaitedReserved }
func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) {
kv, err := s.kv()
if err != nil {
return 0, err
}
st, err := kv.Status(ctx)
if err != nil {
return 0, err
}
return st.Bytes(), nil
}
// entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry.
func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
return func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
@@ -114,21 +96,3 @@ func callWithin(ctx context.Context, call toolCaller, key string, body any) (jso
func publishBody(p versions.Publish) gopackageregistry.PublishInput {
return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version}
}
// republishing starts the publish again once both its states are bound: the queue is taken up again (design 47 G §588). A version's
// publish is asked when its build ends and at the start (A-457a), never waited on a running or manifest-reads-changed.
type republishing struct {
rp *versions.Republisher
ready atomic.Int32
}
func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) func(jetstream.KeyValue) {
return func(jetstream.KeyValue) {
if g.ready.Add(1) != 2 {
return
}
if err := g.rp.Resume(ctx); err != nil {
logf("the queued publishes are not taken up again: %v", err)
}
}
}
@@ -45,12 +45,15 @@ type reservations struct {
// built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none.
built func(ctx context.Context, module, version string) error
released func(ctx context.Context, module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// ended ends the publish item of a build withdrawn, superseded or dropped, its held items terminated; heard acknowledges the
// held items of a version a published newly marked (design 47 H2 §793). Nil are none.
itemEnded, itemHeard func(ctx context.Context, module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
withdraw func(module, fingerprint, why string)
// call asks a seat's verb: the git seat's tag (design 47 G §370).
@@ -146,8 +149,8 @@ func (rv *reservations) took(ctx context.Context, older, newer versions.Reserved
} else {
rv.logf("%s: the reservation of %s (build of %s, never released) is dropped: the build of %s takes the series over and reserves %s", newer.Module, older.Version, older.Fingerprint, newer.Fingerprint, newer.Version)
}
if rv.released != nil {
rv.released(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b)
if rv.itemEnded != nil {
rv.itemEnded(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793)
}
rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series))
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module)
@@ -350,6 +353,15 @@ func (rv *reservations) published(ctx context.Context, o versions.Outcome, inPro
for _, m := range out.Marked {
rv.sayPublished(ctx, m, o.Digest)
}
if rv.itemHeard != nil { // the publish items a published newly marked are acknowledged (§793)
seen := map[string]bool{}
for _, m := range out.Marked {
if k := m.Reserved.Module + " " + m.Reserved.Version; !seen[k] {
seen[k] = true
rv.itemHeard(ctx, m.Reserved.Module, m.Reserved.Version)
}
}
}
}
// sayPublished says the seat mesh-builds' package-published of an artifact a published marked.
@@ -372,8 +384,8 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str
if err != nil {
return "", err
}
if ok && held && rv.released != nil {
rv.released(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b)
if ok && held && rv.itemEnded != nil {
rv.itemEnded(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793)
}
if !ok {
return "", fmt.Errorf("%s holds no reservation of %s", module, series)
@@ -457,6 +457,7 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) {
var asked, ended []string
rv.built = func(_ context.Context, m, v string) error { asked = append(asked, m+" "+v); return nil }
rv.released = func(_ context.Context, m, v string) { ended = append(ended, m+" "+v) }
rv.itemEnded = rv.released // a build withdrawn or superseded ends its publish item too (§793)
registry := []string{"0.1.37"}
rv.r.HeldNumbers = func(context.Context, string, string, []string) ([]string, error) { return registry, nil }
rv.filled.Store(true)
+1
View File
@@ -5,6 +5,7 @@ go 1.26.0
require (
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
@@ -74,6 +74,27 @@ const ModePublish = "publish"
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
const ModeRepublish = "republish"
// ModePublishDrop and ModePublishFirst are the Modes of a reasks value of builder_publish_drop and builder_publish_first (design
// 47 H2 §793): Module and Version name the item; the builder terminates it, or puts it again as live and terminates the old.
const (
ModePublishDrop = "publish-drop"
ModePublishFirst = "publish-first"
)
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) or builder_publish_first (ModePublishFirst) writes
// (§793): its key, `<mode>.<unix nanoseconds>` (a call is one value), and the value. Module, version and why are required.
func NewPublishAct(mode, module, version, why, by string, at time.Time) (string, []byte, error) {
module, version = strings.TrimSpace(module), strings.TrimSpace(version)
if module == "" || version == "" {
return "", nil, errors.New("give the item's module and version")
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why, in plain words (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: mode, Module: module, Version: version})
return mode + "." + strconv.FormatInt(at.UnixNano(), 10), data, err
}
// ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*).
const ModeFloorAgain = "floor-again"
@@ -27,6 +27,10 @@ type Acts struct {
ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error)
// DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not.
DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error)
// PublishDrop terminates one publish item, and PublishFirst puts it again as live and terminates the old (builder_publish_drop,
// builder_publish_first, design 47 H2 §793): the answer, or why not.
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
PublishFirst func(ctx context.Context, module, version, by, why string) (string, error)
}
// FloorAgain is builder_floor_again of name: refused, since the builder numbers nothing of its own and no claim asks a floor
@@ -17,22 +17,28 @@ import (
// **A reserved version of a done build is published by the builder until its release is written** (design 47 G §588, A-457a): the
// publish is an item of the builder's re-ask state, asked when the build ends and at its start, tried with a growing pause and
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has one queue of
// versions, kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has its items on
// the builder's work queue `publishes` (design 47 H2 §793), subject `publish.<holder>.<class>.<module>.<version>`, pulled by one
// consumer per holder and class, the live class first. A send is the registry seat's `publish` with the digest and no
// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. Nothing waits on a
// node-engine `running` or `manifest-reads-changed` any more.
const (
// PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB.
// PublishQueueState is the builder's old state of the queues, whose items are put on the work queue once at the first start
// of this release, then dropped through the bus module (design 47 H2 §793).
PublishQueueState = "publish-queue"
PublishQueueMiB = 4
// PublishesLocal is the builder's work queue of publish items, asked for in its requires (§793, §820; H-A-448).
PublishesLocal = "publishes"
// ActedState holds, one key per subject, the last stream sequence the builder acted on; capped at ActedMiB.
ActedState = "acted"
ActedMiB = 1
// PublishQueueTool is builder_publish_queue.
PublishQueueTool = "builder_publish_queue"
// QueueFull is what version-unreleased says of a version the queue's cap refused.
// QueueFull is what version-unreleased says of a version the work queue's cap refused.
QueueFull = "queue full"
// PublishDropTool and PublishFirstTool are builder_publish_drop and builder_publish_first (§793).
PublishDropTool = "builder_publish_drop"
PublishFirstTool = "builder_publish_first"
// NotStoredWords opens what a refused artifact is said as.
NotStoredWords = "not stored: "
// VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds
@@ -40,11 +46,11 @@ const (
VerbPublish = "publish"
VerbVersions = "versions"
// PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on;
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's queue sends every
// live one before any of the start, each class oldest first (design 47 H H-A-290 (1)).
PublishClassLive = "build-ended-this-run"
PublishClassStart = "asked-again-at-start"
// PublishSendsAtOnce bounds the sends of one holder's queue that run at once (H-A-290 (3)).
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled
// live first, each class oldest first (design 47 H H-A-290 (1), H2 §793).
PublishClassLive = "live"
PublishClassStart = "again"
// PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793).
PublishSendsAtOnce = 4
// PublishQueueListCap caps the queued versions and the publish items builder_publish_queue lists (H-A-290 (5)).
PublishQueueListCap = 50
@@ -77,9 +83,8 @@ func SeatOfArtifact(a meshcatalog.Artifact) (string, bool) {
return seat, ok
}
// QueuedPublish is the value of a publish-queue key `<holder module>.<module>.<version>` (each token escaped, §73): the
// artifacts of the version this holder publishes, by reference with their digests, when it was queued and whether its
// publish has begun.
// QueuedPublish is one item of the work queue publishes (§793), and the value of a key of the old state publish-queue: the
// artifacts of the version this holder publishes, by reference with their digests, and when it was queued.
type QueuedPublish struct {
V int `json:"v"`
Holder string `json:"holder"`
@@ -87,7 +92,6 @@ type QueuedPublish struct {
Version string `json:"version"`
Artifacts map[string]string `json:"artifacts"`
Queued time.Time `json:"queued"`
Begun bool `json:"begun,omitempty"`
// Class is PublishClassLive or PublishClassStart (H-A-290 (1)); a value written before it had one is of the start.
Class string `json:"class,omitempty"`
}
@@ -110,25 +114,31 @@ func sortQueued(qs []QueuedPublish) {
})
}
// QueueFilter is the filter of every queued version: a queue key has three tokens, which AllFilter (two) never lists on the bus.
const QueueFilter = "*.*.*"
// QueueKey is the key of holder's queued version of module.
// QueueKey is the key of holder's queued version of module in the old state publish-queue.
func QueueKey(holder, module, version string) string {
return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version)
}
// PublishMsgID is the message id of holder's item of module's version in class (§793): the work queue takes one item of each
// id within its duplicates window, so an item put twice is put once.
func PublishMsgID(holder, class, module, version string) string {
return EscapeToken(holder) + "." + class + "." + EscapeToken(module) + "." + EscapeToken(version)
}
// PublishEvent is the subject of holder's item of module's version in class within the work queue publishes (§793):
// `publish.<holder>.<class>.<module>.<version>`, each token escaped.
func PublishEvent(holder, class, module, version string) string {
return "publish." + PublishMsgID(holder, class, module, version)
}
// ErrQueueFull is a put the work queue refused as full: said as QueueFull, and asked again at the item's next try (§820).
var ErrQueueFull = errors.New("the work queue " + PublishesLocal + " is full")
// Acted is the value of an acted key: the last stream sequence acted on for its subject.
type Acted struct {
Sequence uint64 `json:"sequence"`
}
// QueueStore is the queue's bucket: compare-and-set, deletion and how many bytes it holds, for its cap.
type QueueStore interface {
ReserveStore
Bytes(ctx context.Context) (uint64, error)
}
// Publish is one artifact's publish again: what the registry seat's publish takes, without parts.
type Publish struct {
Seat, Kind, Name, Version, Digest, Fingerprint, Commit, Transfer string
@@ -137,7 +147,9 @@ type Publish struct {
// Republisher publishes reserved versions again (design 47 G §588).
type Republisher struct {
Reserved *Reserver
Queue QueueStore
// Put puts one item on the work queue publishes under its message id (PublishMsgID): duplicate says the queue holds that
// id within its duplicates window and took nothing; a full queue is ErrQueueFull.
Put func(ctx context.Context, q QueuedPublish) (duplicate bool, err error)
// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
// Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words.
@@ -146,8 +158,6 @@ type Republisher struct {
Unreleased func(ctx context.Context, v ReservedVersion, why string)
Now func() time.Time
Logf func(string, ...any)
// Go starts the sending of a holder's queue; nil starts a goroutine.
Go func(func())
// Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat
// of the assignments takes too, read from the builder's start (design 47 G A-416, A-416a); nil answers none.
Holders func(seat string) []string
@@ -158,11 +168,7 @@ type Republisher struct {
// nil ends none.
Ended func(ctx context.Context, module, version string)
mu sync.Mutex
busy map[string]bool
// relist marks a holder whose pass lists again before its next start-class send: a live publish was queued meanwhile
// (H-A-291 (2)).
relist map[string]bool
mu sync.Mutex
// lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock,
// or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last.
lastQueued time.Time
@@ -292,18 +298,9 @@ func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[stri
return out, nil
}
// drop deletes every queued key of module's version and ends its publish item: it never publishes (H-A-290 (2)), said with why.
// drop ends module's version's publish item: it never publishes (H-A-290 (2)), said with why; its items on the work queue are
// terminated when they are pulled (§793).
func (r *Republisher) drop(ctx context.Context, module, version, why string) {
ctx = context.WithoutCancel(ctx)
keys, err := r.Queue.KeysOf(ctx, "*."+EscapeToken(module)+"."+EscapeToken(version))
if err != nil {
r.logf("publish again: %s %s never publishes (%s), and its queued keys cannot be listed: %v", module, version, why, err)
}
for _, k := range keys {
if err := r.Queue.Delete(ctx, k); err != nil {
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
}
}
r.end(ctx, module, version, why)
}
@@ -338,13 +335,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
sort.SliceStable(vs, func(i, j int) bool { return vs[i].At.After(vs[j].At) })
var queued []string
var errs []error
started := map[string]bool{}
// Every holder given a version is drained, whatever a later version met: a version queued and never drained waits forever.
defer func() {
for holder := range started {
r.drainLater(ctx, holder)
}
}()
for _, v := range vs {
if max > 0 && len(queued) >= max {
break
@@ -396,7 +386,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
return queued, err
}
did = did || ok
started[holder] = true
}
if did {
queued = append(queued, v.Module+" "+v.Version)
@@ -405,42 +394,25 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
return queued, errors.Join(errs...)
}
// enqueue puts one holder's artifacts of a version in its queue: not again when it is queued already, and not past the
// section's cap (said as queue full).
// enqueue puts one holder's artifacts of a version on the work queue under its message id: not again while the queue holds that
// id (its duplicates window), which stands queued, and not past the queue's cap (said as queue full).
func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder string, arts map[string]string) (bool, error) {
key := QueueKey(holder, v.Module, v.Version)
if _, _, found, err := r.Queue.Get(ctx, key); err != nil {
return false, err
} else if found {
r.logf("publish again: %s %s is queued at %s already", v.Module, v.Version, holder)
return false, nil
}
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)})
used, err := r.Queue.Bytes(ctx)
if err != nil {
return false, fmt.Errorf("the size of the state %s cannot be read: %w", PublishQueueState, err)
}
if used+uint64(len(raw)) > PublishQueueMiB<<20 {
q := QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)}
dup, err := r.Put(ctx, q)
switch {
case errors.Is(err, ErrQueueFull):
r.logf("publish again: %s %s is not queued at %s: %v; its publish item asks it again at its next try", v.Module, v.Version, holder, err)
if r.Unreleased != nil {
r.Unreleased(ctx, v, QueueFull)
}
return false, nil
case err != nil:
return false, fmt.Errorf("%s %s cannot be queued at %s: %w", v.Module, v.Version, holder, err)
case dup:
r.logf("publish again: %s %s is queued at %s already (class %s): the work queue holds its message id", v.Module, v.Version, holder, q.Class)
return true, nil
}
if _, err := r.Queue.Create(ctx, key, raw); err != nil {
if errors.Is(err, ErrConflict) {
return false, nil
}
return false, fmt.Errorf("%s cannot be queued: %w", key, err)
}
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), r.classOf(v))
if r.classOf(v) == PublishClassLive { // a pass sending a start list lists again, so this one goes next (H-A-291 (2))
r.mu.Lock()
if r.relist == nil {
r.relist = map[string]bool{}
}
r.relist[holder] = true
r.mu.Unlock()
}
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), q.Class)
return true, nil
}
@@ -537,15 +509,15 @@ func PublishItemKey(module, version string) string {
return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version)
}
// AnswerQueue is builder_publish_queue over the three states as the tools bundle reads them (H-A-290 (5)): each state read in
// bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the
// AnswerQueue is builder_publish_queue over the work queue publishes (its items, read in pages through the tool runtime, §793,
// H-A-448) and the two states as the tools bundle reads them (H-A-290 (5)): each read in bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the
// listed queue to one module's (one version's), so an item's class can be read.
func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) {
out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}}
vals, c, err := queue.values(PublishQueueState)
vals, c, err := queue.values(PublishesLocal)
out.capped(c)
if err != nil {
return out, fmt.Errorf("the state %s cannot be read: %w", PublishQueueState, err)
return out, fmt.Errorf("the work queue %s cannot be read: %w", PublishesLocal, err)
}
counts := map[[2]string]*QueueCount{}
var listed []QueuedPublish
@@ -637,13 +609,21 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e
}
// PublishQueueDescription is builder_publish_queue's description.
const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588, H H-A-290): " +
"how many each registry seat holder's queue holds per class (build-ended-this-run, sent first, or asked-again-at-start) with the oldest and newest " +
"queue time; the first 50 in the order they are sent, each with its class, its artifacts' digests, when it was queued and whether its publish has " +
"begun (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
const PublishQueueDescription = "The versions the builder will publish again (its work queue " + PublishesLocal + ", design 47 G section 588, H2 section 793): " +
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, pulled first, or again, asked again at a start) " +
"with the oldest and newest queue time; the first 50 in the order they are pulled, each with its class, its artifacts' digests and when it was " +
"queued (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
"sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " +
"tries and next try, and why one ended after its tries. Each state is read in one page; a state past it is answered capped, naming the " +
"keys read and the state's size (H-A-293). Read-only."
"tries and next try, and why one ended (its tries spent, or dropped). A read past its pages is answered capped, naming the items read and the " +
"queue's size (H-A-293). Read-only."
// PublishDropDescription and PublishFirstDescription are builder_publish_drop's and builder_publish_first's (§793).
const PublishDropDescription = "Drop one version's publish (design 47 H2 section 793): its items on the work queue " + PublishesLocal + " are " +
"terminated at every holder, and its publish item ended with who dropped it and why, so it is never sent or put again; builder_publish_again " +
"asks it anew. A hand act, recorded in the hand-act log first."
const PublishFirstDescription = "Put one version's publish first (design 47 H2 section 793): each of its again items on the work queue " +
PublishesLocal + " is put again as live, under the message id <holder>.live.<module>.<version>, pulled before every again item, and the old " +
"terminated. A hand act, recorded in the hand-act log first."
// PublishAgainDescription is builder_publish_again's description.
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
@@ -652,17 +632,14 @@ const PublishAgainDescription = "Publish reserved versions again (design 47 G se
"with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " +
"recorded in the hand-act log."
// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588): a version whose reservation
// was withdrawn or taken over is dropped with its publish item ended (H-A-290 (2)), and one of the live class whose build ended
// before this run is of the start's now (H-A-290 (1)). It reads the reservations once.
func (r *Republisher) Resume(ctx context.Context) error {
keys, err := r.Queue.KeysOf(ctx, QueueFilter)
if err != nil {
return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
}
// MoveQueued puts the values of the old state publish-queue on the work queue, once (§793): each under its message id, so a
// move run twice puts each once within the duplicates window; its class as of this run (a build ended before it is of the
// start's, H-A-290 (1)); a version whose reservation was withdrawn, taken over or superseded is not put, its publish item
// ended (H-A-290 (2), H-A-291 (1)). It reads the reservations once, and answers how many it put.
func (r *Republisher) MoveQueued(ctx context.Context, vals []json.RawMessage) (int, error) {
all, err := r.Reserved.All(ctx)
if err != nil {
return fmt.Errorf("the queued publishes are not taken up: %w", err)
return 0, fmt.Errorf("the queued publishes are not moved: %w", err)
}
reserved := make(map[string]ReservedVersion, len(all))
for _, v := range all {
@@ -670,16 +647,12 @@ func (r *Republisher) Resume(ctx context.Context) error {
}
retired, err := r.Reserved.Retired(ctx, all)
if err != nil {
r.logf("publish again: the superseded builds are not retired at the start: %v", err)
r.logf("publish again: the superseded builds are not retired at the move: %v", err)
}
holders := map[string]bool{}
for _, k := range keys {
raw, rev, found, err := r.Queue.Get(ctx, k)
if err != nil || !found {
continue
}
put := 0
for _, raw := range vals {
var q QueuedPublish
if json.Unmarshal(raw, &q) != nil {
if json.Unmarshal(raw, &q) != nil || q.Holder == "" {
continue
}
v, ok := reserved[q.Module+" "+q.Version]
@@ -688,126 +661,38 @@ func (r *Republisher) Resume(ctx context.Context) error {
why = retired[q.Module+" "+q.Version]
}
if why != "" {
if err := r.Queue.Delete(ctx, k); err != nil {
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
}
r.end(ctx, q.Module, q.Version, why)
continue
}
if class := r.classOf(v); q.Class != class {
q.Class = class
if raw, err := json.Marshal(q); err == nil {
if _, err := r.Queue.Update(ctx, k, raw, rev); err != nil {
r.logf("%s keeps its class %s at the start: %v", k, q.Class, err)
}
}
}
holders[q.Holder] = true
}
for h := range holders {
r.drainLater(ctx, h)
}
return nil
}
func (r *Republisher) drainLater(ctx context.Context, holder string) {
r.mu.Lock()
if r.busy == nil {
r.busy = map[string]bool{}
}
if r.busy[holder] {
r.mu.Unlock()
return
}
r.busy[holder] = true
r.mu.Unlock()
run := func() {
defer func() { r.mu.Lock(); delete(r.busy, holder); r.mu.Unlock() }()
r.Drain(ctx, holder)
}
if r.Go != nil {
r.Go(run)
return
}
go run()
}
// Drain sends holder's queued versions in passes (H-A-290 (3)): a pass lists the holder's keys once, reads each once, and sends
// them in their order (the live class first, each class oldest first), PublishSendsAtOnce at a time; the keys are listed again
// only when the pass is spent, or when a live publish was queued while it sends a start list: then the sends in flight end and
// the keys are listed again, so the live one goes next (H-A-291 (2)); a start-class one queued meanwhile waits for the pass's end.
// Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said.
func (r *Republisher) Drain(ctx context.Context, holder string) {
for ctx.Err() == nil {
r.mu.Lock()
delete(r.relist, holder)
r.mu.Unlock()
keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*")
q.Class = r.classOf(v)
dup, err := r.Put(ctx, q)
if err != nil {
r.logf("the queue of %s cannot be listed: %v", holder, err)
return
return put, fmt.Errorf("%s %s is not moved to the work queue %s: %w", q.Module, q.Version, PublishesLocal, err)
}
items := make([]QueuedPublish, 0, len(keys))
for _, k := range keys {
if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found {
var q QueuedPublish
if json.Unmarshal(raw, &q) == nil {
items = append(items, q)
}
}
if !dup {
put++
}
if len(items) == 0 {
return
}
sortQueued(items)
sem := make(chan struct{}, PublishSendsAtOnce)
var wg sync.WaitGroup
for _, q := range items {
if ctx.Err() != nil {
break
}
sem <- struct{}{}
r.mu.Lock()
again := q.Class != PublishClassLive && r.relist[holder]
r.mu.Unlock()
if again {
<-sem
r.logf("publish again: %s's pass lists again after its sends in flight: a live publish was queued", holder)
break
}
wg.Add(1)
go func() {
defer func() { <-sem; wg.Done() }()
r.send(ctx, q)
}()
}
wg.Wait()
}
return put, nil
}
// send publishes one queued version's artifacts and says the outcome, then deletes its key. Every call it makes honours the one
// bound it is given, UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why
// (H-A-290 (4)).
func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
key := QueueKey(q.Holder, q.Module, q.Version)
defer func() {
if err := r.Queue.Delete(context.WithoutCancel(ctx), key); err != nil {
r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err)
}
}()
// SendFate is what SendQueued did with one item (§793): Sent, every artifact sent (its item acknowledged when its publish is
// heard or confirmed); Ended, why the version never publishes (its item terminated); Failed, the holder's or the call's words
// (its item handed back with the pause).
type SendFate struct {
Sent bool
Ended, Failed string
}
// SendQueued publishes one item's artifacts and says the outcome. Every call it makes honours the one bound it is given,
// UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why (H-A-290 (4)).
func (r *Republisher) SendQueued(ctx context.Context, q QueuedPublish) SendFate {
bound, cancel := context.WithTimeout(ctx, UnreleasedBound)
defer cancel()
if !q.Begun {
q.Begun = true
raw, _ := json.Marshal(q)
if _, rev, found, err := r.Queue.Get(bound, key); err == nil && found {
_, _ = r.Queue.Update(bound, key, raw, rev)
}
}
all, err := r.Reserved.All(bound)
if err != nil {
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; its publish item asks it again", q.Module, q.Version, q.Holder, err)
return
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; it is handed back", q.Module, q.Version, q.Holder, err)
return SendFate{Failed: "its reservation cannot be read: " + err.Error()}
}
var v ReservedVersion
found := false
@@ -830,43 +715,64 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
}
if why != "" {
r.end(ctx, q.Module, q.Version, why)
return
return SendFate{Ended: why}
}
e, err := r.Entry(bound, v.Fingerprint)
if err != nil {
r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err))
return
words := fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err)
r.say(ctx, v, words)
return SendFate{Failed: words}
}
missing := map[string]bool{}
for _, ref := range v.Missing() {
missing[ref] = true
}
refs := make([]string, 0, len(q.Artifacts))
for ref := range q.Artifacts {
if _, no := v.Unstored[ref]; no || !missing[ref] {
continue // published already, or refused as not stored: said once, never sent again
}
refs = append(refs, ref)
}
sort.Strings(refs)
if len(refs) == 0 {
return SendFate{Ended: "none of its artifacts is left to send: each is published or refused as not stored"}
}
var failed []string
sent := 0
for _, ref := range refs {
seat, name, kind, ok := publishTarget(e, q.Module, ref)
if !ok {
r.say(ctx, v, ref+": the catalog entry names no registry seat for it")
return
return SendFate{Failed: ref + ": the catalog entry names no registry seat for it"}
}
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))})
if err == nil {
r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref])
sent++
continue
}
if err != nil {
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
words := err.Error()
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
} else if ok {
r.say(ctx, v, ref+" "+NotStoredWords+words)
}
continue
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
words := err.Error()
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
} else if ok {
r.say(ctx, v, ref+" "+NotStoredWords+words)
}
r.say(ctx, v, ref+": "+words)
continue
}
r.say(ctx, v, ref+": "+words)
failed = append(failed, ref+": "+words)
}
switch {
case len(failed) > 0:
return SendFate{Failed: strings.Join(failed, "; ")}
case sent == 0:
return SendFate{Ended: "each artifact left was refused as not stored"}
}
return SendFate{Sent: true}
}
// publishTarget is the registry seat, the name and the kind module's artifact ref is published as, from its catalog entry; ok is
@@ -9,51 +9,40 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
type queueMem struct {
*reserveMem
bytes uint64
// listings counts the listings of one holder's keys (a filter `<holder>.*.*`), what a drain's pass lists.
listings atomic.Int32
}
func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil }
func (q *queueMem) KeysOf(ctx context.Context, filter string) ([]string, error) {
if strings.HasSuffix(filter, ".*.*") && filter != QueueFilter {
q.listings.Add(1)
}
return q.reserveMem.KeysOf(ctx, filter)
}
// republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by
// the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record.
// the Go seat), one holder module `oci` that claims the oci seat, a work queue that records its puts (deduplicating by message
// id when window is set, refusing when full) and sends each put at once as its pull would, and a Send and a Unreleased that
// record.
type republishBed struct {
*Republisher
reserved *reserveMem
queue *queueMem
now time.Time
mu sync.Mutex // sent and said, which a holder's sends at once write
mu sync.Mutex // sent and said, which sends at once write
sent []Publish
said []string
ended []string
refuse string // the words Send fails with, empty for none
// holders are the registry seats' holders the seat-holders view names (design 47 G A-416).
holders map[string][]string
// puts are the items the work queue took; window deduplicates by message id; full refuses every put; held keeps the puts
// unsent.
puts []QueuedPublish
ids map[string]bool
window, full bool
held bool
}
func newRepublishBed(t *testing.T) *republishBed {
t.Helper()
b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)}
b := &republishBed{reserved: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC), ids: map[string]bool{}}
res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Now: func() time.Time { return b.now }, Go: func(f func()) { f() },
b.Republisher = &Republisher{Reserved: res, Now: func() time.Time { return b.now },
Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) {
if fp == "src2:holder" {
return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil
@@ -83,6 +72,21 @@ func newRepublishBed(t *testing.T) *republishBed {
b.ended = append(b.ended, module+" "+version)
},
Logf: t.Logf}
b.Put = func(ctx context.Context, q QueuedPublish) (bool, error) {
if b.full {
return false, fmt.Errorf("%w: maximum bytes exceeded", ErrQueueFull)
}
id := PublishMsgID(q.Holder, q.Class, q.Module, q.Version)
if b.window && b.ids[id] {
return true, nil
}
b.ids[id] = true
b.puts = append(b.puts, q)
if !b.held {
b.SendQueued(ctx, q)
}
return false, nil
}
b.Holders = func(seat string) []string { return b.holders[seat] }
return b
}
@@ -125,8 +129,8 @@ func TestRepublish(t *testing.T) {
p.Transfer != "republish-m-"+strings.ReplaceAll(v.Version, ".", "-")+"-prog" || strings.ContainsAny(p.Transfer, "/. *>") {
t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p)
}
if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 {
t.Errorf("a key stays in the queue after its outcome: %v", keys)
if len(b.puts) != 1 || b.puts[0].Holder != "oci" || b.puts[0].Class != PublishClassLive {
t.Errorf("the work queue took %+v, want one live item at oci", b.puts)
}
})
t.Run("an entry that cannot be read stops neither the other versions nor the drain", func(t *testing.T) {
@@ -173,22 +177,42 @@ func TestRepublish(t *testing.T) {
t.Run("a queue past its cap says queue full", func(t *testing.T) {
b := newRepublishBed(t)
b.reserve(t, "m")
b.queue.bytes = PublishQueueMiB << 20
b.full = true
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
_, _ = b.Republish(ctx, "", "")
if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) {
t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said)
}
})
t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) {
t.Run("an item put twice within the duplicates window is put once, and stands queued", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true})
if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil {
b.window, b.held = true, true
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
q1, _ := b.Republish(ctx, "m", v.Version)
q2, _ := b.Republish(ctx, "m", v.Version)
if len(b.puts) != 1 || len(q1) != 1 || len(q2) != 1 {
t.Errorf("puts %d, queued %v then %v: want one item, queued both times", len(b.puts), q1, q2)
}
})
t.Run("the move puts each item of the old state once, its class as of this run, a superseded one not", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
w := b.reserve(t, "n")
b.Started = b.now
b.window, b.held = true, true
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Class: "build-ended-this-run"})
gone, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "n", Version: w.Version, Artifacts: map[string]string{"n:prog": "sha256:aa"}, Queued: b.now})
if _, _, err := b.Reserved.Withdraw(ctx, "n", w.Version); err != nil {
t.Fatal(err)
}
if err := b.Resume(ctx); err != nil || len(b.sent) != 1 {
t.Errorf("resume sent %d (%v), want the begun version sent anew", len(b.sent), err)
for range 2 {
if _, err := b.MoveQueued(ctx, []json.RawMessage{raw, gone}); err != nil {
t.Fatal(err)
}
}
if len(b.puts) != 1 || b.puts[0].Class != PublishClassStart || fmt.Sprint(b.ended) != "[n "+w.Version+" n "+w.Version+"]" {
t.Errorf("puts %+v, ended %v: want m once as again (its build ended before this run), n's item ended", b.puts, b.ended)
}
})
t.Run("after a fresh start the tool queues at the seat's holder, no running heard", func(t *testing.T) {
@@ -262,98 +286,31 @@ func TestTheRegistriesRecordSaysWhetherAVersionIsHeld(t *testing.T) {
}
// queueAt writes a queued version of module m at holder oci directly, as a start finds it.
func (b *republishBed) queueAt(t *testing.T, v ReservedVersion, class string, at time.Time) {
t.Helper()
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class})
if _, err := b.queue.Create(context.Background(), QueueKey("oci", v.Module, v.Version), raw); err != nil {
t.Fatal(err)
}
func (b *republishBed) queueAt(v ReservedVersion, class string, at time.Time) QueuedPublish {
return QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class}
}
// TestAHoldersQueueSendsLiveFirstInPasses: design 47 H H-A-290 (1)-(3) — a publish whose build ended in this run is sent before
// any of a build that ended before the start, each class oldest first; a pass lists the holder's keys once and sends them, at most
// four at a time; a build whose reservation was withdrawn or taken over never publishes, its key dropped and its publish item ended,
// at enqueue, at the pass and at the start.
func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
// TestAQueuedItemIsSentWhileItsBuildStands: design 47 H H-A-290 (2), H-A-291 (1), H2 §793 — an item is put at the holder in the
// class of its build (live when it ended in this run); a build whose reservation was withdrawn, taken over or superseded by a newer
// patch or release of its series never publishes, its publish item ended, at the put and at the send; an item whose artifacts are
// all published or refused as not stored ends unsent.
func TestAQueuedItemIsSentWhileItsBuildStands(t *testing.T) {
ctx := context.Background()
t.Run("the live class first, at most four sends at once", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
b := newRepublishBed(t)
var old []ReservedVersion
for _, m := range []string{"o1", "o2", "o3", "o4", "o5"} {
old = append(old, b.reserve(t, m))
}
b.Started = b.now
b.reserve(t, "live")
for i, v := range old {
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
}
b.now = b.now.Add(time.Hour) // the live one is queued last, after every old one
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
release := make(chan struct{})
var arrived []string
var inflight, most int
send := b.Send
b.Send = func(ctx context.Context, p Publish) error {
b.mu.Lock()
arrived = append(arrived, p.Name)
inflight++
most = max(most, inflight)
b.mu.Unlock()
<-release
b.mu.Lock()
inflight--
b.mu.Unlock()
return send(ctx, p)
}
b.Go = func(f func()) { go f() }
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
t.Fatalf("queued %v (%v), want the live version", q, err)
}
synctest.Wait() // every send the bound lets run waits on release
b.mu.Lock()
first := slices.Clone(arrived)
b.mu.Unlock()
if len(first) != PublishSendsAtOnce || !slices.Contains(first, "live/prog") {
t.Errorf("the sends running at once are %v: want %d, the live one among them", first, PublishSendsAtOnce)
}
close(release)
synctest.Wait()
if len(b.sent) != 6 || most != PublishSendsAtOnce {
t.Errorf("sent %d, at most %d at once: want all six, %d at once", len(b.sent), most, PublishSendsAtOnce)
}
})
})
t.Run("a pass lists the keys once for all its sends", func(t *testing.T) {
t.Run("a build that ended before this run is put as again, one after it as live", func(t *testing.T) {
b := newRepublishBed(t)
var vs []ReservedVersion
for i := range 6 {
vs = append(vs, b.reserve(t, fmt.Sprintf("m%d", i)))
}
for i, v := range vs {
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
}
b.Drain(ctx, "oci")
if n := b.queue.listings.Load(); len(b.sent) != 6 || n != 2 {
t.Errorf("sent %d with %d listings, want six sends from one pass and one listing that finds it spent", len(b.sent), n)
}
})
t.Run("a start makes a live key of a build before it a start key", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
old := b.reserve(t, "o")
b.Started = b.now
b.queueAt(t, v, PublishClassLive, b.now)
b.Go = func(func()) {} // the start's drain is not run here
if err := b.Resume(ctx); err != nil {
t.Fatal(err)
}
raw, _, _, _ := b.queue.Get(ctx, QueueKey("oci", "m", v.Version))
var q QueuedPublish
if json.Unmarshal(raw, &q) != nil || q.Class != PublishClassStart {
t.Errorf("the key after the start is %s, want the class %s: its build did not end in this run", raw, PublishClassStart)
live := b.reserve(t, "l")
b.held = true
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
_, _ = b.Republish(ctx, "o", old.Version)
_, _ = b.Republish(ctx, "l", live.Version)
if len(b.puts) != 2 || b.puts[0].Class != PublishClassStart || b.puts[1].Class != PublishClassLive ||
PublishEvent("oci", PublishClassLive, "l", live.Version) != "publish.oci.live.l.0=2E1=2E0" {
t.Errorf("puts %+v, want o again and l live, on publish.<holder>.<class>.<module>.<version>", b.puts)
}
})
for _, at := range []string{"enqueue", "pass", "start"} {
for _, at := range []string{"put", "send"} {
t.Run("a withdrawn build never publishes, at the "+at, func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
@@ -361,105 +318,34 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
if _, ok, err := b.Reserved.Withdraw(ctx, "m", v.Version); err != nil || !ok {
t.Fatalf("withdrawn %v (%v)", ok, err)
}
switch at {
case "enqueue":
b.queueAt(t, v, PublishClassLive, b.now)
b.Go = func(func()) {} // a key queued before the withdrawal is dropped by the enqueue itself
if at == "put" {
if q, _ := b.Republish(ctx, "m", v.Version); len(q) != 0 {
t.Errorf("queued %v", q)
}
case "pass":
b.queueAt(t, v, PublishClassLive, b.now)
b.Drain(ctx, "oci")
case "start":
b.queueAt(t, v, PublishClassLive, b.now)
b.Go = func(func()) {}
if err := b.Resume(ctx); err != nil {
t.Fatal(err)
}
} else if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" {
t.Errorf("the send of a withdrawn build answered %+v, want it ended", f)
}
keys, _ := b.queue.KeysOf(ctx, QueueFilter)
if len(b.sent) != 0 || len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("sent %v, keys %v, items ended %v: want nothing sent, the key dropped and m %s's item ended", b.sent, keys, b.ended, v.Version)
if len(b.sent) != 0 || len(b.puts) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("sent %v, puts %v, items ended %v: want nothing sent or put and m %s's item ended", b.sent, b.puts, b.ended, v.Version)
}
})
}
t.Run("a live publish queued mid-pass goes after the sends in flight, before the rest of the start list", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
b := newRepublishBed(t)
var old []ReservedVersion
for _, m := range []string{"o1", "o2", "o3", "o4", "o5", "o6"} {
old = append(old, b.reserve(t, m))
}
b.Started = b.now
b.reserve(t, "live")
for i, v := range old {
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
}
b.now = b.now.Add(time.Hour)
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
first, then := make(chan struct{}), make(chan struct{}) // the four in flight wait on first, every later send on then
var arrived []string
send := b.Send
b.Send = func(ctx context.Context, p Publish) error {
b.mu.Lock()
arrived = append(arrived, p.Name)
gate := map[bool]chan struct{}{true: first, false: then}[len(arrived) <= PublishSendsAtOnce]
b.mu.Unlock()
<-gate
return send(ctx, p)
}
b.Go = func(f func()) { go f() }
b.drainLater(ctx, "oci")
synctest.Wait() // four start sends in flight, the pass holding o5 and o6
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
t.Fatalf("queued %v (%v)", q, err)
}
close(first)
synctest.Wait() // the sends after the four: with the live one among them, or o5 and o6 alone when the pass goes on
b.mu.Lock()
next := slices.Clone(arrived[PublishSendsAtOnce:])
b.mu.Unlock()
if !slices.Contains(next, "live/prog") {
t.Errorf("after the four in flight the sends are %v, want the live one among them, the pass listed again", next)
}
close(then)
synctest.Wait()
if len(arrived) != 7 {
t.Errorf("sent %v, want all seven", arrived)
}
})
})
t.Run("a build a newer patch or release of its series supersedes is retired at the pass and at the start, never by a newer series", func(t *testing.T) {
t.Run("a build a newer patch or release of its series supersedes is ended at the send, never by a newer series", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil {
t.Fatal(err)
}
b.queueAt(t, v, PublishClassStart, b.now)
b.Drain(ctx, "oci")
if len(b.sent) != 1 || len(b.ended) != 0 {
t.Errorf("with a standing 0.2: sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", b.sent, b.ended, v.Version)
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); !f.Sent || len(b.sent) != 1 || len(b.ended) != 0 {
t.Errorf("with a standing 0.2: %+v, sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", f, b.sent, b.ended, v.Version)
}
rel := &reserveMem{mem: newMem()}
raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.3"}, {Fingerprint: "g", Version: "0.1.9"}}})
_, _ = rel.Create(ctx, ReleasedKey("m"), raw)
b.Reserved.Releases = rel
b.sent = nil
b.queueAt(t, v, PublishClassStart, b.now)
b.Drain(ctx, "oci")
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("at the pass: sent %v, ended %v, want m %s retired by the release 0.1.9", b.sent, b.ended, v.Version)
}
b.ended = nil
b.queueAt(t, v, PublishClassStart, b.now)
b.Go = func(func()) {}
if err := b.Resume(ctx); err != nil {
t.Fatal(err)
}
if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version)
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("at the send: %+v, sent %v, ended %v, want m %s retired by the release 0.1.9", f, b.sent, b.ended, v.Version)
}
only := &reserveMem{mem: newMem()}
raw, _ = json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.0"}}})
@@ -473,13 +359,27 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
b.queueAt(t, v, PublishClassStart, b.now)
if _, err := b.Reserved.Reserve(ctx, "m", "0.1", []string{"m:prog"}, Source{Fingerprint: "src2:other"}); err != nil {
t.Fatal(err)
}
b.Drain(ctx, "oci")
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("sent %v, ended %v: want nothing sent and the item ended", b.sent, b.ended)
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("%+v, sent %v, ended %v: want nothing sent and the item ended", f, b.sent, b.ended)
}
})
t.Run("an item whose artifacts are published or refused as not stored ends unsent", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
b.refuse = "not stored: no blob"
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 {
t.Errorf("a not stored refusal answered %+v, sent %d: want it ended after one send", f, len(b.sent))
}
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 {
t.Errorf("again: %+v, sent %d: want it ended unsent", f, len(b.sent))
}
b.refuse = "the holder is full"
w := b.reserve(t, "n")
if f := b.SendQueued(ctx, b.queueAt(w, PublishClassLive, b.now)); f.Failed == "" || f.Sent {
t.Errorf("a refusal in other words answered %+v, want it failed, handed back", f)
}
})
}
@@ -527,7 +427,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
t.Errorf("a state one page holds is answered capped: %+v", a.Read)
}
queue.All = func() (StatePage, error) { return StatePage{Values: vals, More: true, Keys: 5000, Bytes: 3 << 20}, nil }
if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishQueueState ||
if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishesLocal ||
c.Read[0].KeysRead != 2000 || c.Read[0].StateKeys != 5000 || c.Read[0].StateBytes != 3<<20 || c.Total != 2000 {
t.Errorf("a state past its one page answered capped %v, read %+v (%v): want capped, 2000 keys read of 5000, its size said (H-A-293)", c.Capped, c.Read, err)
}