builder: remove the old publish-queue bucket itself after the move (F1)
The bus module's keeper made no entry for the builder's old-section publish-queue, so nats_drop refused it at every start and the move re-put its items after the 24 h window. The builder, which made the bucket, now deletes it itself once every item is on the work queue; the manifest no longer names publish-queue nor invokes nats.nats_drop. The whole test proves the move and the removal on the embedded bus, and a second move of the same items after a stopped start puts each once. Also corrects the publishes comment: the process consumes the work queue on its own account (H-A-454 second addendum). Builds: design 47 H2 §793, H-A-461 (1), H-A-454 (second addendum)
This commit is contained in:
@@ -353,7 +353,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := pq.learnAll(ctx); err != nil {
|
||||
logger.Printf("the publishes waiting on the work queue %s are not pulled yet: %v; each holder's are pulled when an item of it is put", versions.PublishesLocal, err)
|
||||
}
|
||||
moveOld(ctx, js, own.queue, cred.Module, republish, calls.Call, logger.Printf)
|
||||
moveOld(ctx, js, own.queue, republish, logger.Printf)
|
||||
}()
|
||||
errs := make(chan error, 10)
|
||||
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466)
|
||||
|
||||
+27
-8
@@ -5,6 +5,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
@@ -137,6 +138,11 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
type oldItem struct {
|
||||
key string
|
||||
raw []byte
|
||||
}
|
||||
var olds []oldItem
|
||||
for _, m := range []string{"m1", "m2", "m3"} {
|
||||
v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m})
|
||||
if err != nil {
|
||||
@@ -149,6 +155,7 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw})
|
||||
}
|
||||
rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) {
|
||||
data, _ := json.Marshal(q)
|
||||
@@ -158,13 +165,25 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
}
|
||||
return ack.Duplicate, nil
|
||||
}, Started: time.Now(), Logf: t.Logf}
|
||||
var drops []string
|
||||
drop := func(tool string, body any) (json.RawMessage, error) {
|
||||
drops = append(drops, fmt.Sprint(tool, body))
|
||||
return json.RawMessage(`{}`), nil
|
||||
oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)
|
||||
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
|
||||
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err)
|
||||
}
|
||||
for range 2 {
|
||||
moveOld(ctx, b.js, builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState), "mesh-delivery-builder", rp, drop, t.Logf)
|
||||
// A start that stopped before the removal moves the same items again: the duplicates window keeps each once.
|
||||
again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range olds {
|
||||
if _, err := again.Put(ctx, e.key, e.raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
|
||||
moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves
|
||||
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Errorf("after the second move the old state's bucket answers %v, want it removed", err)
|
||||
}
|
||||
st, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
@@ -174,7 +193,7 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 || len(drops) != 2 {
|
||||
t.Errorf("the move run twice left %v on the work queue, dropped %v: want each of the three once, as again, and the old state dropped through the bus module", info.State.Subjects, drops)
|
||||
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 {
|
||||
t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -528,8 +528,9 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
|
||||
}
|
||||
|
||||
// publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-454): a stream of
|
||||
// the manifest's old streams section with the work-queue retention, the module's own, consumed by the process on the module's
|
||||
// account; each item on `mesh.stream.<module>.event.publish.<holder>.<class>.<module>.<version>` under the message id
|
||||
// the manifest's old streams section with the work-queue retention, the module's own, consumed by the process on its own account
|
||||
// (H-A-454 second addendum), which names it in its bus queues; each item on
|
||||
// `mesh.stream.<module>.event.publish.<holder>.<class>.<module>.<version>` under the message id
|
||||
// `<holder>.<class>.<module>.<version>`. The builder makes the pull consumers at run time as it learns a holder,
|
||||
// `publishes-<holder>-<class>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live
|
||||
// class first, at most four sends at once per holder.
|
||||
@@ -908,9 +909,10 @@ func (p *publishItems) endItem(ctx context.Context, module, version, answer stri
|
||||
}
|
||||
|
||||
// moveOld puts the items of the old state publish-queue on the work queue once at the start (§793): each under its message id, so
|
||||
// a move run twice puts each once within the duplicates window; then the bucket is dropped through the bus module (its tool
|
||||
// nats_drop, on the operator's warrant). A builder whose bus holds no such bucket moves nothing.
|
||||
func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string, rp *versions.Republisher, call toolCaller, logf func(string, ...any)) {
|
||||
// a move run twice puts each once within the duplicates window; then the builder, which made that bucket, removes it itself
|
||||
// (H-A-461 (1): the bus module's keeper holds no entry for it, so its drop refuses it). A builder whose bus holds no such
|
||||
// bucket moves nothing; a move that did not put every item keeps the bucket for the next start.
|
||||
func moveOld(ctx context.Context, js jetstream.JetStream, bucket string, rp *versions.Republisher, logf func(string, ...any)) {
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return
|
||||
@@ -932,22 +934,16 @@ func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string,
|
||||
}
|
||||
n, err := rp.MoveQueued(ctx, vals)
|
||||
if err != nil {
|
||||
logf("the old state %s is not dropped: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
|
||||
logf("the old state %s is not removed: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
|
||||
return
|
||||
}
|
||||
logf("the old state %s: %d of its %d items put on the work queue %s; it is dropped through the bus module", versions.PublishQueueState, n, len(vals), versions.PublishesLocal)
|
||||
if call == nil {
|
||||
if err := js.DeleteKeyValue(ctx, bucket); err != nil && !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
logf("the old state %s: %d of its %d items put on the work queue %s, but its bucket is not removed: %v; the next start moves it again", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
|
||||
return
|
||||
}
|
||||
if _, err := call(natsDropTool, map[string]any{"module": module, "local": versions.PublishQueueState,
|
||||
"why": "its items are on the builder's work queue " + versions.PublishesLocal + " (design 47 H2 section 793)"}); err != nil {
|
||||
logf("the old state %s is not dropped through the bus module: %v", versions.PublishQueueState, err)
|
||||
}
|
||||
logf("the old state %s: %d of its %d items put on the work queue %s; its bucket is removed", versions.PublishQueueState, n, len(vals), versions.PublishesLocal)
|
||||
}
|
||||
|
||||
// natsDropTool is the bus module's drop of an entry no accepted manifest asks for (H-A-126), which the old state goes by.
|
||||
const natsDropTool = "nats.nats_drop"
|
||||
|
||||
// publishQueueVerb is the handler of the seat mesh-builds' verb publish-queue (§793; H-A-454): the builder reads its own work queue
|
||||
// and answers its counts per holder and class, the oldest and newest, and a capped list; builder_publish_queue asks it.
|
||||
func publishQueueVerb(ctx context.Context, pq *publishQueue) process.SeatHandler {
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
|
||||
const (
|
||||
// PublishQueueState is the builder's old state of the queues, whose items are put on the work queue once at the first start
|
||||
// of this release, then dropped through the bus module (design 47 H2 §793).
|
||||
// of this release, then removed by the builder itself, which made it (design 47 H2 §793; H-A-461 (1)).
|
||||
PublishQueueState = "publish-queue"
|
||||
// PublishesLocal is the builder's work queue of publish items, asked for in its requires (§793, §820; H-A-448).
|
||||
PublishesLocal = "publishes"
|
||||
|
||||
@@ -122,7 +122,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5, MaxMiB: 128}, {Name: "engine-reads", History: 1, MaxMiB: 8}, {Name: "findings", History: 5, MaxMiB: 32}, {Name: "floor-waits", History: 1, MaxMiB: 8}, {Name: "hearing", History: 1, MaxMiB: 8}, {Name: "instances", History: 1, MaxMiB: 8}, {Name: "old-form", History: 1, MaxMiB: 8}, {Name: "bounds", History: 1, KeyTTL: true, MaxMiB: 16}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1, MaxMiB: 8}, {Name: "toolchains-heard", History: 1, MaxMiB: 8}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1, MaxMiB: 8}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1, MaxMiB: 8}, {Name: "reasks", History: 1, MaxMiB: 8}, {Name: "versions", History: 1, MaxMiB: 32}, {Name: "first-views", History: 1, MaxMiB: 8}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory // caps per H-A-437 over the fills read on novox 2026-11-22 (design 47 H2 §812, §814)
|
||||
if want := []state{{Name: "builds", History: 5, MaxMiB: 128}, {Name: "engine-reads", History: 1, MaxMiB: 8}, {Name: "findings", History: 5, MaxMiB: 32}, {Name: "floor-waits", History: 1, MaxMiB: 8}, {Name: "hearing", History: 1, MaxMiB: 8}, {Name: "instances", History: 1, MaxMiB: 8}, {Name: "old-form", History: 1, MaxMiB: 8}, {Name: "bounds", History: 1, KeyTTL: true, MaxMiB: 16}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1, MaxMiB: 8}, {Name: "toolchains-heard", History: 1, MaxMiB: 8}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1, MaxMiB: 8}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1, MaxMiB: 8}, {Name: "reasks", History: 1, MaxMiB: 8}, {Name: "versions", History: 1, MaxMiB: 32}, {Name: "first-views", History: 1, MaxMiB: 8}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory // caps per H-A-437 over the fills read on novox 2026-11-22 (design 47 H2 §812, §814)
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
@@ -156,7 +156,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
|
||||
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions",
|
||||
// A-231: the build asks on each build seat's queue delivery (accept.delivery.<asker>).
|
||||
"seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery", "nats.nats_drop", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) {
|
||||
"seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no
|
||||
|
||||
@@ -126,7 +126,6 @@
|
||||
"seat:node-build-go.accept.delivery",
|
||||
"seat:node-build-node.accept.delivery",
|
||||
"seat:node-build-image.accept.delivery",
|
||||
"nats.nats_drop",
|
||||
"seat:node-build-go.current",
|
||||
"seat:node-build-image.current",
|
||||
"seat:node-build-node.current"
|
||||
@@ -145,7 +144,6 @@
|
||||
{ "name": "toolchains-heard", "history": 1, "max-mib": 8 },
|
||||
{ "name": "waits", "history": 1, "max-mib": 4 },
|
||||
{ "name": "reserved", "history": 1, "max-mib": 8 },
|
||||
{ "name": "publish-queue", "history": 1, "max-mib": 4 },
|
||||
{ "name": "acted", "history": 1, "max-mib": 1 },
|
||||
{ "name": "asked", "history": 1, "max-mib": 8 },
|
||||
{ "name": "reasks", "history": 1, "max-mib": 8 },
|
||||
@@ -417,7 +415,6 @@
|
||||
"toolchains-heard",
|
||||
"waits",
|
||||
"reserved",
|
||||
"publish-queue",
|
||||
"acted",
|
||||
"asked",
|
||||
"reasks",
|
||||
@@ -462,8 +459,7 @@
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:node-build-go.accept.delivery",
|
||||
"seat:node-build-node.accept.delivery",
|
||||
"seat:node-build-image.accept.delivery",
|
||||
"nats.nats_drop"
|
||||
"seat:node-build-image.accept.delivery"
|
||||
]
|
||||
},
|
||||
"health": {
|
||||
|
||||
Reference in New Issue
Block a user