diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go index 9b28d4df9..75768393d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go @@ -1,15 +1,10 @@ package main import ( - "encoding/json" - "fmt" - "sort" - "strings" "time" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/health" - "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-builder/internal/versions" ) @@ -17,54 +12,29 @@ import ( const ( // HealthTool is builder_health, the builder's health as its manifest's health asks it (design 47 H2 §794, H-A-435). HealthTool = "builder_health" - // BehindAfter is how long a durable consumer's oldest unacknowledged message may wait before the health names it: an - // events backlog, or a handler standing still on one message (§794). - BehindAfter = 5 * time.Minute + // eventsBehindBoundSetting is the setting events-behind-bound (§794; H-A-461 (2), P11): how long a durable consumer's + // oldest unacknowledged message may wait before the health names it, an events backlog or a handler standing still on one + // message; defaultEventsBehindBound when none is set. + eventsBehindBoundSetting = "events-behind-bound" + defaultEventsBehindBound = 5 * time.Minute ) // healthTool is builder_health: healthy when it answers, and each of the builder's durable consumers' oldest unacknowledged -// message waits less than BehindAfter, as the consumers' facts the builder writes at every delivery and every acknowledgement -// say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise unhealthy, naming each -// consumer behind and how long. +// message waits less than the setting events-behind-bound, as the consumers' facts the builder writes at every delivery and +// every acknowledgement say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise +// unhealthy, naming each consumer behind and how long. A setting that holds no duration is said as the health's answer. func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool { return stdio.Tool{Name: HealthTool, Description: "The builder's health: healthy when it answers and each of its durable consumers' oldest unacknowledged message waits less " + - "than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " + - "Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.", + "than the setting events-behind-bound (five minutes by default; no events backlog, no handler standing still on a message); otherwise it " + + "names each consumer behind and how long. Read from the builder's state consumers, which it writes at every message delivered and " + + "acknowledged. Read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return builderHealth(consumers, now()), nil }} -} - -// builderHealth is builder_health's answer at now. -func builderHealth(consumers versions.Reader, now time.Time) health.Answer { - keys, err := consumers.Keys() - if err != nil { - return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err)) - } - if len(keys) == 0 { - return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState)) - } - sort.Strings(keys) - var behind []string - for _, k := range keys { - raw, ok, err := consumers.Get(k) - if err != nil { - return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err)) - } - var f process.ConsumerFacts - if ok { - if err := json.Unmarshal(raw, &f); err != nil { - return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err)) + Run: func(map[string]any) (any, error) { + bound, err := stdio.SettingDuration(eventsBehindBoundSetting, defaultEventsBehindBound) + if err != nil { + return health.Unhealthy("The builder's " + err.Error() + "."), nil } - } - if !f.OldestUnacknowledged.IsZero() { - if age := now.Sub(f.OldestUnacknowledged); age >= BehindAfter { - behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending)) - } - } - } - if len(behind) > 0 { - return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", BehindAfter, strings.Join(behind, ", "))) - } - return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter)) + return versions.BuilderHealth(consumers, now(), bound), nil + }} } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go index 49c5e349c..f02bd22ca 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" "mesh-delivery-builder/internal/build" @@ -58,13 +59,23 @@ func TestAStateIsReadInOnePage(t *testing.T) { "builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)} r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil }, Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }} - a := builderHealth(r, now) + a := versions.BuilderHealth(r, now, defaultEventsBehindBound) if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." { t.Errorf("health = %+v, want unhealthy naming builder-events alone", a) } - if a := builderHealth(r, now.Add(-2*time.Minute)); !a.Healthy { + if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), defaultEventsBehindBound); !a.Healthy { t.Errorf("health four minutes in = %+v, want healthy", a) } + // H-A-461 (2): the bound is the setting events-behind-bound, five minutes when none is set. + tool := healthTool(r, func() time.Time { return now }) + for _, c := range []struct{ set, want string }{{"", "unhealthy"}, {"10m", "healthy"}, {"soon", "events-behind-bound"}} { + t.Setenv(stdio.SettingWord(eventsBehindBoundSetting), c.set) + got, _ := tool.Run(nil) + a := got.(health.Answer) + if (c.want == "healthy") != a.Healthy || (c.want == "events-behind-bound" && !strings.Contains(a.Why, c.want)) { + t.Errorf("with events-behind-bound %q the health = %+v, want %s", c.set, a, c.want) + } + } }) // Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from // its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go index 5f073bc21..ff7d41ed3 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go @@ -1,14 +1,30 @@ package main import ( + "errors" "time" "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" ) -// SlowEvent is how long the handling of one heard event may take before the builder says it in its journal (design 47 H2 -// §794): the line names the event's subject, the part that took it and how long it took. -const SlowEvent = 100 * time.Millisecond +// slowEventSetting is the setting slow-event (design 47 H2 §794; H-A-461 (2), P11): how long the handling of one heard event +// may take before the builder says it in its journal, the line naming the event's subject, the part that took it and how long +// it took; defaultSlowEvent when none is set. +const ( + slowEventSetting = "slow-event" + defaultSlowEvent = 100 * time.Millisecond +) + +// slowEvent reads the setting slow-event. +func slowEvent() (time.Duration, error) { + d, err := stdio.SettingDuration(slowEventSetting, defaultSlowEvent) + if err != nil { + return 0, errors.New("the builder's " + err.Error()) + } + return d, nil +} // eventTaker is one part of the builder that takes events off its durable consumer: take answers whether the message was its. type eventTaker struct { @@ -16,9 +32,9 @@ type eventTaker struct { take func(jetstream.Msg) bool } -// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed SlowEvent -// (design 47 H2 §794). now is the clock, a test's when set. -func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() time.Time, logf func(string, ...any)) { +// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed slow, the +// setting slow-event (design 47 H2 §794). now is the clock, a test's when set. +func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, slow time.Duration, now func() time.Time, logf func(string, ...any)) { if now == nil { now = time.Now } @@ -33,7 +49,7 @@ func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() if part == rest.part { rest.take(m) } - if took := now().Sub(start); took > SlowEvent { - logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), SlowEvent, part) + if took := now().Sub(start); took > slow { + logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), slow, part) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 0a4b3bdb4..faf0ad6c0 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -353,7 +353,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := pq.learnAll(ctx); err != nil { logger.Printf("the publishes waiting on the work queue %s are not pulled yet: %v; each holder's are pulled when an item of it is put", versions.PublishesLocal, err) } - moveOld(ctx, js, own.queue, cred.Module, republish, calls.Call, logger.Printf) + moveOld(ctx, js, own.queue, republish, logger.Printf) }() errs := make(chan error, 10) b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466) @@ -479,7 +479,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf} // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} - // Each part takes the events it hears in turn; a handling over SlowEvent is said with the part that took it (design 47 H2 §794). + // Each part takes the events it hears in turn; a handling over the setting slow-event is said with the part that took it + // (design 47 H2 §794; H-A-461 (2)). + slow, err := slowEvent() + if err != nil { + return err + } takers := []eventTaker{ {"the operator's answer to an ask about a key", decided.heard}, // §568 {"the retake of a key failed on a transport fault", func(m jetstream.Msg) bool { return retake.heard(ctx, m, logger.Printf) }}, // §568 @@ -491,7 +496,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro {"the registry seats' outcomes", func(m jetstream.Msg) bool { return seatHeard.heard(ctx, m) }}, } gone := eventTaker{"the artifacts gone", func(m jetstream.Msg) bool { heardGone(ctx, b, m, logger.Printf); return true }} - events := func(m jetstream.Msg) { takeEvent(m, takers, gone, nil, logger.Printf) } + events := func(m jetstream.Msg) { takeEvent(m, takers, gone, slow, nil, logger.Printf) } // The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, the indexer's take-in.taken and // manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the // catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index 6e9f4c770..0ea8fa90f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -5,26 +5,33 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "slices" + "strings" "sync" "testing" "time" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-builder/internal/versions" ) // Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder. // A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and -// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and asks the -// bus module to drop the old state. The seat mesh-builds' verb publish-queue reads the stream: counts per holder and class, the -// oldest and newest, the live item listed first (H-A-454). +// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the +// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts +// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit and +// §794's backlog held six minutes, as steps (H-A-453 (3)). func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { b := startBed(t) ctx := b.ctx @@ -131,12 +138,26 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { if err != nil { t.Fatal(err) } - call := func(string, any) (json.RawMessage, error) { + var tagsWritten []string + call := func(tool string, body any) (json.RawMessage, error) { + switch tool { + case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796) + return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}}) + case "seat:" + gitseat.Seat + ".tag": + tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit) + return json.Marshal(gitseat.TagAnswer{Performed: true}) + } return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) } + found := &memFindings{vals: map[string][]byte{}} rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, - &findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf) + &findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf) rv.filled.Store(true) + type oldItem struct { + key string + raw []byte + } + var olds []oldItem for _, m := range []string{"m1", "m2", "m3"} { v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m}) if err != nil { @@ -149,6 +170,7 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil { t.Fatal(err) } + olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw}) } rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) { data, _ := json.Marshal(q) @@ -158,13 +180,25 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { } return ack.Duplicate, nil }, Started: time.Now(), Logf: t.Logf} - var drops []string - drop := func(tool string, body any) (json.RawMessage, error) { - drops = append(drops, fmt.Sprint(tool, body)) - return json.RawMessage(`{}`), nil + oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState) + moveOld(ctx, b.js, oldBucket, rp, t.Logf) + if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { + t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err) } - for range 2 { - moveOld(ctx, b.js, builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState), "mesh-delivery-builder", rp, drop, t.Logf) + // A start that stopped before the removal moves the same items again: the duplicates window keeps each once. + again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket}) + if err != nil { + t.Fatal(err) + } + for _, e := range olds { + if _, err := again.Put(ctx, e.key, e.raw); err != nil { + t.Fatal(err) + } + } + moveOld(ctx, b.js, oldBucket, rp, t.Logf) + moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves + if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { + t.Errorf("after the second move the old state's bucket answers %v, want it removed", err) } st, err := b.js.Stream(ctx, bind.Stream) if err != nil { @@ -174,7 +208,87 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { if err != nil { t.Fatal(err) } - if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 || len(drops) != 2 { - t.Errorf("the move run twice left %v on the work queue, dropped %v: want each of the three once, as again, and the old state dropped through the bus module", info.State.Subjects, drops) + if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 { + t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects) + } + + // Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another + // commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags /v at + // the build's commit, and the module's version-released clears the finding. + rv.r.Names = func(context.Context, string, string) (string, bool, error) { + return "git.example/o/r/modules/m4", true, nil + } + v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib", + Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1}) + elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere) + if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised { + t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised) + } + catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events + if err != nil { + t.Fatal(err) + } + evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy, + FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy}) + if err != nil { + t.Fatal(err) + } + next := func(module, version string) jetstream.Msg { + t.Helper() + data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version}) + if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil { + t.Fatal(err) + } + m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second)) + if err != nil { + t.Fatal(err) + } + return m + } + if !rv.heard(ctx, next("m4", "0.2.1")) { + t.Fatal("the reservations did not take version-released") + } + if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) { + t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten) + } + + // Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its + // consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five + // minutes it does not. + consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)}) + if err != nil { + t.Fatal(err) + } + held := time.Now() + hub := &oneHandler{} + aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held }, + sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}} + _ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again + hub.handle(next("m5", "0.1.0")) + state := versions.Reader{Keys: func() ([]string, error) { + l, err := consumers.ListKeys(ctx) + if err != nil { + return nil, err + } + var keys []string + for k := range l.Keys() { + keys = append(keys, k) + } + return keys, nil + }, Get: func(k string) (json.RawMessage, bool, error) { + e, err := consumers.Get(ctx, k) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil + }} + if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") { + t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a) + } + if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy { + t.Errorf("the health four minutes in = %+v, want healthy", a) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index 86ffb3e2c..30eabf196 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -528,8 +528,9 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion { } // publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-454): a stream of -// the manifest's old streams section with the work-queue retention, the module's own, consumed by the process on the module's -// account; each item on `mesh.stream..event.publish....` under the message id +// the manifest's old streams section with the work-queue retention, the module's own, consumed by the process on its own account +// (H-A-454 second addendum), which names it in its bus queues; each item on +// `mesh.stream..event.publish....` under the message id // `...`. The builder makes the pull consumers at run time as it learns a holder, // `publishes--`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live // class first, at most four sends at once per holder. @@ -908,9 +909,10 @@ func (p *publishItems) endItem(ctx context.Context, module, version, answer stri } // moveOld puts the items of the old state publish-queue on the work queue once at the start (§793): each under its message id, so -// a move run twice puts each once within the duplicates window; then the bucket is dropped through the bus module (its tool -// nats_drop, on the operator's warrant). A builder whose bus holds no such bucket moves nothing. -func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string, rp *versions.Republisher, call toolCaller, logf func(string, ...any)) { +// a move run twice puts each once within the duplicates window; then the builder, which made that bucket, removes it itself +// (H-A-461 (1): the bus module's keeper holds no entry for it, so its drop refuses it). A builder whose bus holds no such +// bucket moves nothing; a move that did not put every item keeps the bucket for the next start. +func moveOld(ctx context.Context, js jetstream.JetStream, bucket string, rp *versions.Republisher, logf func(string, ...any)) { kv, err := js.KeyValue(ctx, bucket) if errors.Is(err, jetstream.ErrBucketNotFound) { return @@ -932,22 +934,16 @@ func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string, } n, err := rp.MoveQueued(ctx, vals) if err != nil { - logf("the old state %s is not dropped: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err) + logf("the old state %s is not removed: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err) return } - logf("the old state %s: %d of its %d items put on the work queue %s; it is dropped through the bus module", versions.PublishQueueState, n, len(vals), versions.PublishesLocal) - if call == nil { + if err := js.DeleteKeyValue(ctx, bucket); err != nil && !errors.Is(err, jetstream.ErrBucketNotFound) { + logf("the old state %s: %d of its %d items put on the work queue %s, but its bucket is not removed: %v; the next start moves it again", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err) return } - if _, err := call(natsDropTool, map[string]any{"module": module, "local": versions.PublishQueueState, - "why": "its items are on the builder's work queue " + versions.PublishesLocal + " (design 47 H2 section 793)"}); err != nil { - logf("the old state %s is not dropped through the bus module: %v", versions.PublishQueueState, err) - } + logf("the old state %s: %d of its %d items put on the work queue %s; its bucket is removed", versions.PublishQueueState, n, len(vals), versions.PublishesLocal) } -// natsDropTool is the bus module's drop of an entry no accepted manifest asks for (H-A-126), which the old state goes by. -const natsDropTool = "nats.nats_drop" - // publishQueueVerb is the handler of the seat mesh-builds' verb publish-queue (§793; H-A-454): the builder reads its own work queue // and answers its counts per holder and class, the oldest and newest, and a capped list; builder_publish_queue asks it. func publishQueueVerb(ctx context.Context, pq *publishQueue) process.SeatHandler { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 6d744c14d..874e41f9f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -16,6 +16,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" @@ -587,11 +588,27 @@ func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) { slow := []eventTaker{{"the reservations", func(jetstream.Msg) bool { clock = clock.Add(150 * time.Millisecond); return true }}} fast := []eventTaker{{"the floor", func(jetstream.Msg) bool { clock = clock.Add(100 * time.Millisecond); return true }}} rest := eventTaker{"the artifacts gone", func(jetstream.Msg) bool { return true }} - takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) - takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) + takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) + takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) if len(said) != 1 || said[0] != "the event a.published took 150ms to handle, over 100ms: the reservations took it" { t.Errorf("said %q, want one line for the handling over 100 ms", said) } + // H-A-461 (2): the bound is the setting slow-event, 100 ms when none is set; at 200ms the 150 ms handling is not said. + for _, c := range []struct { + set string + want time.Duration + }{{"", defaultSlowEvent}, {"200ms", 200 * time.Millisecond}, {"soon", 0}} { + t.Setenv(stdio.SettingWord(slowEventSetting), c.set) + if d, err := slowEvent(); d != c.want || (c.want == 0) != (err != nil) { + t.Errorf("with slow-event %q the bound is %v (%v), want %v", c.set, d, err, c.want) + } + } + said = nil + takeEvent(&agedTestMsg{subject: "a.published", seq: 3}, slow, rest, 200*time.Millisecond, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) + if len(said) != 0 { + t.Errorf("said %q with slow-event 200ms, want nothing for a 150 ms handling", said) + } + clock = clock.Add(-150 * time.Millisecond) // the ledger's times below read as before this row facts := map[string]string{} l := &ledgers{now: now, sink: func(_ context.Context, k string, v []byte) error { facts[k] = string(v); return nil }} hub := &oneHandler{} diff --git a/modules/mesh-delivery-builder/internal/versions/health.go b/modules/mesh-delivery-builder/internal/versions/health.go new file mode 100644 index 000000000..22d2c9193 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/health.go @@ -0,0 +1,49 @@ +package versions + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// BuilderHealth is builder_health's answer at now (design 47 H2 §794, H-A-435, H-A-462): healthy when each of the builder's +// durable consumers' oldest unacknowledged message waits less than behindAfter, as the consumers' facts the builder writes at +// every delivery and every acknowledgement say them (its state consumers, read through consumers); otherwise unhealthy, naming +// each consumer behind and how long. The judgement is the module's shared code, so the builder's whole test proves it. +func BuilderHealth(consumers Reader, now time.Time, behindAfter time.Duration) health.Answer { + keys, err := consumers.Keys() + if err != nil { + return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err)) + } + if len(keys) == 0 { + return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState)) + } + sort.Strings(keys) + var behind []string + for _, k := range keys { + raw, ok, err := consumers.Get(k) + if err != nil { + return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err)) + } + var f process.ConsumerFacts + if ok { + if err := json.Unmarshal(raw, &f); err != nil { + return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err)) + } + } + if !f.OldestUnacknowledged.IsZero() { + if age := now.Sub(f.OldestUnacknowledged); age >= behindAfter { + behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending)) + } + } + } + if len(behind) > 0 { + return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", behindAfter, strings.Join(behind, ", "))) + } + return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), behindAfter)) +} diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index 3719c65ce..4fa84f819 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -26,7 +26,7 @@ import ( const ( // PublishQueueState is the builder's old state of the queues, whose items are put on the work queue once at the first start - // of this release, then dropped through the bus module (design 47 H2 §793). + // of this release, then removed by the builder itself, which made it (design 47 H2 §793; H-A-461 (1)). PublishQueueState = "publish-queue" // PublishesLocal is the builder's work queue of publish items, asked for in its requires (§793, §820; H-A-448). PublishesLocal = "publishes" diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index dbf4dd856..5e552a268 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -122,7 +122,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5, MaxMiB: 128}, {Name: "engine-reads", History: 1, MaxMiB: 8}, {Name: "findings", History: 5, MaxMiB: 32}, {Name: "floor-waits", History: 1, MaxMiB: 8}, {Name: "hearing", History: 1, MaxMiB: 8}, {Name: "instances", History: 1, MaxMiB: 8}, {Name: "old-form", History: 1, MaxMiB: 8}, {Name: "bounds", History: 1, KeyTTL: true, MaxMiB: 16}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1, MaxMiB: 8}, {Name: "toolchains-heard", History: 1, MaxMiB: 8}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1, MaxMiB: 8}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1, MaxMiB: 8}, {Name: "reasks", History: 1, MaxMiB: 8}, {Name: "versions", History: 1, MaxMiB: 32}, {Name: "first-views", History: 1, MaxMiB: 8}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory // caps per H-A-437 over the fills read on novox 2026-11-22 (design 47 H2 §812, §814) + if want := []state{{Name: "builds", History: 5, MaxMiB: 128}, {Name: "engine-reads", History: 1, MaxMiB: 8}, {Name: "findings", History: 5, MaxMiB: 32}, {Name: "floor-waits", History: 1, MaxMiB: 8}, {Name: "hearing", History: 1, MaxMiB: 8}, {Name: "instances", History: 1, MaxMiB: 8}, {Name: "old-form", History: 1, MaxMiB: 8}, {Name: "bounds", History: 1, KeyTTL: true, MaxMiB: 16}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1, MaxMiB: 8}, {Name: "toolchains-heard", History: 1, MaxMiB: 8}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1, MaxMiB: 8}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1, MaxMiB: 8}, {Name: "reasks", History: 1, MaxMiB: 8}, {Name: "versions", History: 1, MaxMiB: 32}, {Name: "first-views", History: 1, MaxMiB: 8}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory // caps per H-A-437 over the fills read on novox 2026-11-22 (design 47 H2 §812, §814) t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -156,7 +156,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions", // A-231: the build asks on each build seat's queue delivery (accept.delivery.). - "seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery", "nats.nats_drop", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) { + "seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no @@ -200,10 +200,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 { t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams) } - if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 10 || + if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 || // design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" || - !m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) || + !m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || + // H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default + m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime || + m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) || m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 { t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index fb8837287..ffbd8b463 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -126,7 +126,6 @@ "seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery", - "nats.nats_drop", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current" @@ -145,7 +144,6 @@ { "name": "toolchains-heard", "history": 1, "max-mib": 8 }, { "name": "waits", "history": 1, "max-mib": 4 }, { "name": "reserved", "history": 1, "max-mib": 8 }, - { "name": "publish-queue", "history": 1, "max-mib": 4 }, { "name": "acted", "history": 1, "max-mib": 1 }, { "name": "asked", "history": 1, "max-mib": 8 }, { "name": "reasks", "history": 1, "max-mib": 8 }, @@ -206,6 +204,20 @@ "default": "30s", "runtime": true }, + "slow-event": { + "type": "string", + "description": "how long the handling of one heard event may take before the builder says it in its journal, with its duration and the part that took it, a Go duration (design 47 H2 §794; H-A-461 (2)).", + "trusted": false, + "default": "100ms", + "runtime": true + }, + "events-behind-bound": { + "type": "string", + "description": "how long a durable consumer's oldest unacknowledged message may wait before builder_health names it, an events backlog or a handler standing still, a Go duration (design 47 H2 §794; H-A-461 (2)).", + "trusted": false, + "default": "5m", + "runtime": true + }, "publish-pause": { "type": "number", "description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).", @@ -417,7 +429,6 @@ "toolchains-heard", "waits", "reserved", - "publish-queue", "acted", "asked", "reasks", @@ -462,8 +473,7 @@ "seat:mesh-delivery.source", "seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", - "seat:node-build-image.accept.delivery", - "nats.nats_drop" + "seat:node-build-image.accept.delivery" ] }, "health": {