Join: merge feat/47-H2-f1 into feat/47-H2

This commit is contained in:
2026-11-22 03:39:45 +01:00
11 changed files with 291 additions and 100 deletions
@@ -1,15 +1,10 @@
package main
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/versions"
)
@@ -17,54 +12,29 @@ import (
const (
// HealthTool is builder_health, the builder's health as its manifest's health asks it (design 47 H2 §794, H-A-435).
HealthTool = "builder_health"
// BehindAfter is how long a durable consumer's oldest unacknowledged message may wait before the health names it: an
// events backlog, or a handler standing still on one message (§794).
BehindAfter = 5 * time.Minute
// eventsBehindBoundSetting is the setting events-behind-bound (§794; H-A-461 (2), P11): how long a durable consumer's
// oldest unacknowledged message may wait before the health names it, an events backlog or a handler standing still on one
// message; defaultEventsBehindBound when none is set.
eventsBehindBoundSetting = "events-behind-bound"
defaultEventsBehindBound = 5 * time.Minute
)
// healthTool is builder_health: healthy when it answers, and each of the builder's durable consumers' oldest unacknowledged
// message waits less than BehindAfter, as the consumers' facts the builder writes at every delivery and every acknowledgement
// say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise unhealthy, naming each
// consumer behind and how long.
// message waits less than the setting events-behind-bound, as the consumers' facts the builder writes at every delivery and
// every acknowledgement say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise
// unhealthy, naming each consumer behind and how long. A setting that holds no duration is said as the health's answer.
func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool {
return stdio.Tool{Name: HealthTool,
Description: "The builder's health: healthy when it answers and each of its durable consumers' oldest unacknowledged message waits less " +
"than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " +
"Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.",
"than the setting events-behind-bound (five minutes by default; no events backlog, no handler standing still on a message); otherwise it " +
"names each consumer behind and how long. Read from the builder's state consumers, which it writes at every message delivered and " +
"acknowledged. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return builderHealth(consumers, now()), nil }}
}
// builderHealth is builder_health's answer at now.
func builderHealth(consumers versions.Reader, now time.Time) health.Answer {
keys, err := consumers.Keys()
if err != nil {
return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err))
}
if len(keys) == 0 {
return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState))
}
sort.Strings(keys)
var behind []string
for _, k := range keys {
raw, ok, err := consumers.Get(k)
if err != nil {
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err))
}
var f process.ConsumerFacts
if ok {
if err := json.Unmarshal(raw, &f); err != nil {
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err))
Run: func(map[string]any) (any, error) {
bound, err := stdio.SettingDuration(eventsBehindBoundSetting, defaultEventsBehindBound)
if err != nil {
return health.Unhealthy("The builder's " + err.Error() + "."), nil
}
}
if !f.OldestUnacknowledged.IsZero() {
if age := now.Sub(f.OldestUnacknowledged); age >= BehindAfter {
behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending))
}
}
}
if len(behind) > 0 {
return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", BehindAfter, strings.Join(behind, ", ")))
}
return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter))
return versions.BuilderHealth(consumers, now(), bound), nil
}}
}
@@ -11,6 +11,7 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-builder/internal/build"
@@ -58,13 +59,23 @@ func TestAStateIsReadInOnePage(t *testing.T) {
"builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)}
r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil },
Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }}
a := builderHealth(r, now)
a := versions.BuilderHealth(r, now, defaultEventsBehindBound)
if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." {
t.Errorf("health = %+v, want unhealthy naming builder-events alone", a)
}
if a := builderHealth(r, now.Add(-2*time.Minute)); !a.Healthy {
if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), defaultEventsBehindBound); !a.Healthy {
t.Errorf("health four minutes in = %+v, want healthy", a)
}
// H-A-461 (2): the bound is the setting events-behind-bound, five minutes when none is set.
tool := healthTool(r, func() time.Time { return now })
for _, c := range []struct{ set, want string }{{"", "unhealthy"}, {"10m", "healthy"}, {"soon", "events-behind-bound"}} {
t.Setenv(stdio.SettingWord(eventsBehindBoundSetting), c.set)
got, _ := tool.Run(nil)
a := got.(health.Answer)
if (c.want == "healthy") != a.Healthy || (c.want == "events-behind-bound" && !strings.Contains(a.Why, c.want)) {
t.Errorf("with events-behind-bound %q the health = %+v, want %s", c.set, a, c.want)
}
}
})
// Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and
@@ -1,14 +1,30 @@
package main
import (
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// SlowEvent is how long the handling of one heard event may take before the builder says it in its journal (design 47 H2
// §794): the line names the event's subject, the part that took it and how long it took.
const SlowEvent = 100 * time.Millisecond
// slowEventSetting is the setting slow-event (design 47 H2 §794; H-A-461 (2), P11): how long the handling of one heard event
// may take before the builder says it in its journal, the line naming the event's subject, the part that took it and how long
// it took; defaultSlowEvent when none is set.
const (
slowEventSetting = "slow-event"
defaultSlowEvent = 100 * time.Millisecond
)
// slowEvent reads the setting slow-event.
func slowEvent() (time.Duration, error) {
d, err := stdio.SettingDuration(slowEventSetting, defaultSlowEvent)
if err != nil {
return 0, errors.New("the builder's " + err.Error())
}
return d, nil
}
// eventTaker is one part of the builder that takes events off its durable consumer: take answers whether the message was its.
type eventTaker struct {
@@ -16,9 +32,9 @@ type eventTaker struct {
take func(jetstream.Msg) bool
}
// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed SlowEvent
// (design 47 H2 §794). now is the clock, a test's when set.
func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() time.Time, logf func(string, ...any)) {
// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed slow, the
// setting slow-event (design 47 H2 §794). now is the clock, a test's when set.
func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, slow time.Duration, now func() time.Time, logf func(string, ...any)) {
if now == nil {
now = time.Now
}
@@ -33,7 +49,7 @@ func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func()
if part == rest.part {
rest.take(m)
}
if took := now().Sub(start); took > SlowEvent {
logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), SlowEvent, part)
if took := now().Sub(start); took > slow {
logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), slow, part)
}
}
@@ -353,7 +353,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := pq.learnAll(ctx); err != nil {
logger.Printf("the publishes waiting on the work queue %s are not pulled yet: %v; each holder's are pulled when an item of it is put", versions.PublishesLocal, err)
}
moveOld(ctx, js, own.queue, cred.Module, republish, calls.Call, logger.Printf)
moveOld(ctx, js, own.queue, republish, logger.Printf)
}()
errs := make(chan error, 10)
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466)
@@ -479,7 +479,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf}
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
// Each part takes the events it hears in turn; a handling over SlowEvent is said with the part that took it (design 47 H2 §794).
// Each part takes the events it hears in turn; a handling over the setting slow-event is said with the part that took it
// (design 47 H2 §794; H-A-461 (2)).
slow, err := slowEvent()
if err != nil {
return err
}
takers := []eventTaker{
{"the operator's answer to an ask about a key", decided.heard}, // §568
{"the retake of a key failed on a transport fault", func(m jetstream.Msg) bool { return retake.heard(ctx, m, logger.Printf) }}, // §568
@@ -491,7 +496,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
{"the registry seats' outcomes", func(m jetstream.Msg) bool { return seatHeard.heard(ctx, m) }},
}
gone := eventTaker{"the artifacts gone", func(m jetstream.Msg) bool { heardGone(ctx, b, m, logger.Printf); return true }}
events := func(m jetstream.Msg) { takeEvent(m, takers, gone, nil, logger.Printf) }
events := func(m jetstream.Msg) { takeEvent(m, takers, gone, slow, nil, logger.Printf) }
// The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, the indexer's take-in.taken and
// manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the
// catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has
@@ -5,26 +5,33 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-builder/internal/versions"
)
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder.
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and asks the
// bus module to drop the old state. The seat mesh-builds' verb publish-queue reads the stream: counts per holder and class, the
// oldest and newest, the live item listed first (H-A-454).
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the
// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts
// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit and
// §794's backlog held six minutes, as steps (H-A-453 (3)).
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
b := startBed(t)
ctx := b.ctx
@@ -131,12 +138,26 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
call := func(string, any) (json.RawMessage, error) {
var tagsWritten []string
call := func(tool string, body any) (json.RawMessage, error) {
switch tool {
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796)
return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}})
case "seat:" + gitseat.Seat + ".tag":
tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit)
return json.Marshal(gitseat.TagAnswer{Performed: true})
}
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
}
found := &memFindings{vals: map[string][]byte{}}
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf)
&findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf)
rv.filled.Store(true)
type oldItem struct {
key string
raw []byte
}
var olds []oldItem
for _, m := range []string{"m1", "m2", "m3"} {
v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m})
if err != nil {
@@ -149,6 +170,7 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil {
t.Fatal(err)
}
olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw})
}
rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) {
data, _ := json.Marshal(q)
@@ -158,13 +180,25 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
}
return ack.Duplicate, nil
}, Started: time.Now(), Logf: t.Logf}
var drops []string
drop := func(tool string, body any) (json.RawMessage, error) {
drops = append(drops, fmt.Sprint(tool, body))
return json.RawMessage(`{}`), nil
oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err)
}
for range 2 {
moveOld(ctx, b.js, builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState), "mesh-delivery-builder", rp, drop, t.Logf)
// A start that stopped before the removal moves the same items again: the duplicates window keeps each once.
again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket})
if err != nil {
t.Fatal(err)
}
for _, e := range olds {
if _, err := again.Put(ctx, e.key, e.raw); err != nil {
t.Fatal(err)
}
}
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
t.Errorf("after the second move the old state's bucket answers %v, want it removed", err)
}
st, err := b.js.Stream(ctx, bind.Stream)
if err != nil {
@@ -174,7 +208,87 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 || len(drops) != 2 {
t.Errorf("the move run twice left %v on the work queue, dropped %v: want each of the three once, as again, and the old state dropped through the bus module", info.State.Subjects, drops)
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 {
t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects)
}
// Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another
// commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags <directory>/v<version> at
// the build's commit, and the module's version-released clears the finding.
rv.r.Names = func(context.Context, string, string) (string, bool, error) {
return "git.example/o/r/modules/m4", true, nil
}
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib",
Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1})
elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere)
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised {
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised)
}
catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events
if err != nil {
t.Fatal(err)
}
evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy,
FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy})
if err != nil {
t.Fatal(err)
}
next := func(module, version string) jetstream.Msg {
t.Helper()
data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version})
if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil {
t.Fatal(err)
}
m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second))
if err != nil {
t.Fatal(err)
}
return m
}
if !rv.heard(ctx, next("m4", "0.2.1")) {
t.Fatal("the reservations did not take version-released")
}
if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) {
t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten)
}
// Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its
// consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five
// minutes it does not.
consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)})
if err != nil {
t.Fatal(err)
}
held := time.Now()
hub := &oneHandler{}
aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held },
sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}}
_ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again
hub.handle(next("m5", "0.1.0"))
state := versions.Reader{Keys: func() ([]string, error) {
l, err := consumers.ListKeys(ctx)
if err != nil {
return nil, err
}
var keys []string
for k := range l.Keys() {
keys = append(keys, k)
}
return keys, nil
}, Get: func(k string) (json.RawMessage, bool, error) {
e, err := consumers.Get(ctx, k)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, false, nil
}
if err != nil {
return nil, false, err
}
return e.Value(), true, nil
}}
if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") {
t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a)
}
if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy {
t.Errorf("the health four minutes in = %+v, want healthy", a)
}
}
@@ -528,8 +528,9 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
}
// publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-454): a stream of
// the manifest's old streams section with the work-queue retention, the module's own, consumed by the process on the module's
// account; each item on `mesh.stream.<module>.event.publish.<holder>.<class>.<module>.<version>` under the message id
// the manifest's old streams section with the work-queue retention, the module's own, consumed by the process on its own account
// (H-A-454 second addendum), which names it in its bus queues; each item on
// `mesh.stream.<module>.event.publish.<holder>.<class>.<module>.<version>` under the message id
// `<holder>.<class>.<module>.<version>`. The builder makes the pull consumers at run time as it learns a holder,
// `publishes-<holder>-<class>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live
// class first, at most four sends at once per holder.
@@ -908,9 +909,10 @@ func (p *publishItems) endItem(ctx context.Context, module, version, answer stri
}
// moveOld puts the items of the old state publish-queue on the work queue once at the start (§793): each under its message id, so
// a move run twice puts each once within the duplicates window; then the bucket is dropped through the bus module (its tool
// nats_drop, on the operator's warrant). A builder whose bus holds no such bucket moves nothing.
func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string, rp *versions.Republisher, call toolCaller, logf func(string, ...any)) {
// a move run twice puts each once within the duplicates window; then the builder, which made that bucket, removes it itself
// (H-A-461 (1): the bus module's keeper holds no entry for it, so its drop refuses it). A builder whose bus holds no such
// bucket moves nothing; a move that did not put every item keeps the bucket for the next start.
func moveOld(ctx context.Context, js jetstream.JetStream, bucket string, rp *versions.Republisher, logf func(string, ...any)) {
kv, err := js.KeyValue(ctx, bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
return
@@ -932,22 +934,16 @@ func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string,
}
n, err := rp.MoveQueued(ctx, vals)
if err != nil {
logf("the old state %s is not dropped: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
logf("the old state %s is not removed: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
return
}
logf("the old state %s: %d of its %d items put on the work queue %s; it is dropped through the bus module", versions.PublishQueueState, n, len(vals), versions.PublishesLocal)
if call == nil {
if err := js.DeleteKeyValue(ctx, bucket); err != nil && !errors.Is(err, jetstream.ErrBucketNotFound) {
logf("the old state %s: %d of its %d items put on the work queue %s, but its bucket is not removed: %v; the next start moves it again", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
return
}
if _, err := call(natsDropTool, map[string]any{"module": module, "local": versions.PublishQueueState,
"why": "its items are on the builder's work queue " + versions.PublishesLocal + " (design 47 H2 section 793)"}); err != nil {
logf("the old state %s is not dropped through the bus module: %v", versions.PublishQueueState, err)
}
logf("the old state %s: %d of its %d items put on the work queue %s; its bucket is removed", versions.PublishQueueState, n, len(vals), versions.PublishesLocal)
}
// natsDropTool is the bus module's drop of an entry no accepted manifest asks for (H-A-126), which the old state goes by.
const natsDropTool = "nats.nats_drop"
// publishQueueVerb is the handler of the seat mesh-builds' verb publish-queue (§793; H-A-454): the builder reads its own work queue
// and answers its counts per holder and class, the oldest and newest, and a capped list; builder_publish_queue asks it.
func publishQueueVerb(ctx context.Context, pq *publishQueue) process.SeatHandler {
@@ -16,6 +16,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
@@ -587,11 +588,27 @@ func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) {
slow := []eventTaker{{"the reservations", func(jetstream.Msg) bool { clock = clock.Add(150 * time.Millisecond); return true }}}
fast := []eventTaker{{"the floor", func(jetstream.Msg) bool { clock = clock.Add(100 * time.Millisecond); return true }}}
rest := eventTaker{"the artifacts gone", func(jetstream.Msg) bool { return true }}
takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
if len(said) != 1 || said[0] != "the event a.published took 150ms to handle, over 100ms: the reservations took it" {
t.Errorf("said %q, want one line for the handling over 100 ms", said)
}
// H-A-461 (2): the bound is the setting slow-event, 100 ms when none is set; at 200ms the 150 ms handling is not said.
for _, c := range []struct {
set string
want time.Duration
}{{"", defaultSlowEvent}, {"200ms", 200 * time.Millisecond}, {"soon", 0}} {
t.Setenv(stdio.SettingWord(slowEventSetting), c.set)
if d, err := slowEvent(); d != c.want || (c.want == 0) != (err != nil) {
t.Errorf("with slow-event %q the bound is %v (%v), want %v", c.set, d, err, c.want)
}
}
said = nil
takeEvent(&agedTestMsg{subject: "a.published", seq: 3}, slow, rest, 200*time.Millisecond, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
if len(said) != 0 {
t.Errorf("said %q with slow-event 200ms, want nothing for a 150 ms handling", said)
}
clock = clock.Add(-150 * time.Millisecond) // the ledger's times below read as before this row
facts := map[string]string{}
l := &ledgers{now: now, sink: func(_ context.Context, k string, v []byte) error { facts[k] = string(v); return nil }}
hub := &oneHandler{}
@@ -0,0 +1,49 @@
package versions
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
)
// BuilderHealth is builder_health's answer at now (design 47 H2 §794, H-A-435, H-A-462): healthy when each of the builder's
// durable consumers' oldest unacknowledged message waits less than behindAfter, as the consumers' facts the builder writes at
// every delivery and every acknowledgement say them (its state consumers, read through consumers); otherwise unhealthy, naming
// each consumer behind and how long. The judgement is the module's shared code, so the builder's whole test proves it.
func BuilderHealth(consumers Reader, now time.Time, behindAfter time.Duration) health.Answer {
keys, err := consumers.Keys()
if err != nil {
return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err))
}
if len(keys) == 0 {
return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState))
}
sort.Strings(keys)
var behind []string
for _, k := range keys {
raw, ok, err := consumers.Get(k)
if err != nil {
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err))
}
var f process.ConsumerFacts
if ok {
if err := json.Unmarshal(raw, &f); err != nil {
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err))
}
}
if !f.OldestUnacknowledged.IsZero() {
if age := now.Sub(f.OldestUnacknowledged); age >= behindAfter {
behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending))
}
}
}
if len(behind) > 0 {
return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", behindAfter, strings.Join(behind, ", ")))
}
return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), behindAfter))
}
@@ -26,7 +26,7 @@ import (
const (
// PublishQueueState is the builder's old state of the queues, whose items are put on the work queue once at the first start
// of this release, then dropped through the bus module (design 47 H2 §793).
// of this release, then removed by the builder itself, which made it (design 47 H2 §793; H-A-461 (1)).
PublishQueueState = "publish-queue"
// PublishesLocal is the builder's work queue of publish items, asked for in its requires (§793, §820; H-A-448).
PublishesLocal = "publishes"
@@ -122,7 +122,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.Module != "mesh-delivery-builder" {
t.Errorf("module = %q", m.Module)
}
if want := []state{{Name: "builds", History: 5, MaxMiB: 128}, {Name: "engine-reads", History: 1, MaxMiB: 8}, {Name: "findings", History: 5, MaxMiB: 32}, {Name: "floor-waits", History: 1, MaxMiB: 8}, {Name: "hearing", History: 1, MaxMiB: 8}, {Name: "instances", History: 1, MaxMiB: 8}, {Name: "old-form", History: 1, MaxMiB: 8}, {Name: "bounds", History: 1, KeyTTL: true, MaxMiB: 16}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1, MaxMiB: 8}, {Name: "toolchains-heard", History: 1, MaxMiB: 8}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1, MaxMiB: 8}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1, MaxMiB: 8}, {Name: "reasks", History: 1, MaxMiB: 8}, {Name: "versions", History: 1, MaxMiB: 32}, {Name: "first-views", History: 1, MaxMiB: 8}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory // caps per H-A-437 over the fills read on novox 2026-11-22 (design 47 H2 §812, §814)
if want := []state{{Name: "builds", History: 5, MaxMiB: 128}, {Name: "engine-reads", History: 1, MaxMiB: 8}, {Name: "findings", History: 5, MaxMiB: 32}, {Name: "floor-waits", History: 1, MaxMiB: 8}, {Name: "hearing", History: 1, MaxMiB: 8}, {Name: "instances", History: 1, MaxMiB: 8}, {Name: "old-form", History: 1, MaxMiB: 8}, {Name: "bounds", History: 1, KeyTTL: true, MaxMiB: 16}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1, MaxMiB: 8}, {Name: "toolchains-heard", History: 1, MaxMiB: 8}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1, MaxMiB: 8}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1, MaxMiB: 8}, {Name: "reasks", History: 1, MaxMiB: 8}, {Name: "versions", History: 1, MaxMiB: 32}, {Name: "first-views", History: 1, MaxMiB: 8}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory // caps per H-A-437 over the fills read on novox 2026-11-22 (design 47 H2 §812, §814)
t.Errorf("state = %+v, want %+v", m.State, want)
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
@@ -156,7 +156,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions",
// A-231: the build asks on each build seat's queue delivery (accept.delivery.<asker>).
"seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery", "nats.nats_drop", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) {
"seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no
@@ -200,10 +200,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 {
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
}
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 10 ||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 ||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime ||
// H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default
m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime ||
m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 {
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
}
+15 -5
View File
@@ -126,7 +126,6 @@
"seat:node-build-go.accept.delivery",
"seat:node-build-node.accept.delivery",
"seat:node-build-image.accept.delivery",
"nats.nats_drop",
"seat:node-build-go.current",
"seat:node-build-image.current",
"seat:node-build-node.current"
@@ -145,7 +144,6 @@
{ "name": "toolchains-heard", "history": 1, "max-mib": 8 },
{ "name": "waits", "history": 1, "max-mib": 4 },
{ "name": "reserved", "history": 1, "max-mib": 8 },
{ "name": "publish-queue", "history": 1, "max-mib": 4 },
{ "name": "acted", "history": 1, "max-mib": 1 },
{ "name": "asked", "history": 1, "max-mib": 8 },
{ "name": "reasks", "history": 1, "max-mib": 8 },
@@ -206,6 +204,20 @@
"default": "30s",
"runtime": true
},
"slow-event": {
"type": "string",
"description": "how long the handling of one heard event may take before the builder says it in its journal, with its duration and the part that took it, a Go duration (design 47 H2 §794; H-A-461 (2)).",
"trusted": false,
"default": "100ms",
"runtime": true
},
"events-behind-bound": {
"type": "string",
"description": "how long a durable consumer's oldest unacknowledged message may wait before builder_health names it, an events backlog or a handler standing still, a Go duration (design 47 H2 §794; H-A-461 (2)).",
"trusted": false,
"default": "5m",
"runtime": true
},
"publish-pause": {
"type": "number",
"description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).",
@@ -417,7 +429,6 @@
"toolchains-heard",
"waits",
"reserved",
"publish-queue",
"acted",
"asked",
"reasks",
@@ -462,8 +473,7 @@
"seat:mesh-delivery.source",
"seat:node-build-go.accept.delivery",
"seat:node-build-node.accept.delivery",
"seat:node-build-image.accept.delivery",
"nats.nats_drop"
"seat:node-build-image.accept.delivery"
]
},
"health": {