builder: prove the tag at another commit and the six-minute backlog whole (F2)
The review found §794's backlog and §796's tag at another commit proven only by unit tests. Both become steps of the builder's whole test on the embedded bus: a reservation skips 0.2.0 whose tag stands at another commit, takes 0.2.1 with the finding version-tagged-elsewhere, the release writes modules/m4/v0.2.1 at the build's commit and the version-released heard from the bus clears the finding; a message the handler asks again is kept on its consumer's ledger in the state consumers on the bus, and the health held six minutes names it. The health judgement moves from the tools program's main into internal/versions (BuilderHealth), which the tools program calls, so the whole test can prove it. The test's comment no longer says the old state is dropped through the bus module (F1). Builds: design 47 H2 §794, §796, H-A-453 (3), H-A-435, H-A-462
This commit is contained in:
@@ -1,15 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
@@ -32,39 +26,5 @@ func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool {
|
||||
"than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " +
|
||||
"Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return builderHealth(consumers, now()), nil }}
|
||||
}
|
||||
|
||||
// builderHealth is builder_health's answer at now.
|
||||
func builderHealth(consumers versions.Reader, now time.Time) health.Answer {
|
||||
keys, err := consumers.Keys()
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err))
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState))
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var behind []string
|
||||
for _, k := range keys {
|
||||
raw, ok, err := consumers.Get(k)
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err))
|
||||
}
|
||||
var f process.ConsumerFacts
|
||||
if ok {
|
||||
if err := json.Unmarshal(raw, &f); err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err))
|
||||
}
|
||||
}
|
||||
if !f.OldestUnacknowledged.IsZero() {
|
||||
if age := now.Sub(f.OldestUnacknowledged); age >= BehindAfter {
|
||||
behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", BehindAfter, strings.Join(behind, ", ")))
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter))
|
||||
Run: func(map[string]any) (any, error) { return versions.BuilderHealth(consumers, now(), BehindAfter), nil }}
|
||||
}
|
||||
|
||||
@@ -58,11 +58,11 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
"builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)}
|
||||
r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil },
|
||||
Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }}
|
||||
a := builderHealth(r, now)
|
||||
a := versions.BuilderHealth(r, now, BehindAfter)
|
||||
if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." {
|
||||
t.Errorf("health = %+v, want unhealthy naming builder-events alone", a)
|
||||
}
|
||||
if a := builderHealth(r, now.Add(-2*time.Minute)); !a.Healthy {
|
||||
if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), BehindAfter); !a.Healthy {
|
||||
t.Errorf("health four minutes in = %+v, want healthy", a)
|
||||
}
|
||||
})
|
||||
|
||||
+100
-5
@@ -8,24 +8,30 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder.
|
||||
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
|
||||
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and asks the
|
||||
// bus module to drop the old state. The seat mesh-builds' verb publish-queue reads the stream: counts per holder and class, the
|
||||
// oldest and newest, the live item listed first (H-A-454).
|
||||
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the
|
||||
// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts
|
||||
// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit and
|
||||
// §794's backlog held six minutes, as steps (H-A-453 (3)).
|
||||
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
b := startBed(t)
|
||||
ctx := b.ctx
|
||||
@@ -132,11 +138,20 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
call := func(string, any) (json.RawMessage, error) {
|
||||
var tagsWritten []string
|
||||
call := func(tool string, body any) (json.RawMessage, error) {
|
||||
switch tool {
|
||||
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796)
|
||||
return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}})
|
||||
case "seat:" + gitseat.Seat + ".tag":
|
||||
tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit)
|
||||
return json.Marshal(gitseat.TagAnswer{Performed: true})
|
||||
}
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
||||
}
|
||||
found := &memFindings{vals: map[string][]byte{}}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
||||
&findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
type oldItem struct {
|
||||
key string
|
||||
@@ -196,4 +211,84 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 {
|
||||
t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects)
|
||||
}
|
||||
|
||||
// Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another
|
||||
// commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags <directory>/v<version> at
|
||||
// the build's commit, and the module's version-released clears the finding.
|
||||
rv.r.Names = func(context.Context, string, string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/m4", true, nil
|
||||
}
|
||||
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib",
|
||||
Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1})
|
||||
elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere)
|
||||
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised {
|
||||
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised)
|
||||
}
|
||||
catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := func(module, version string) jetstream.Msg {
|
||||
t.Helper()
|
||||
data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version})
|
||||
if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
if !rv.heard(ctx, next("m4", "0.2.1")) {
|
||||
t.Fatal("the reservations did not take version-released")
|
||||
}
|
||||
if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) {
|
||||
t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten)
|
||||
}
|
||||
|
||||
// Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its
|
||||
// consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five
|
||||
// minutes it does not.
|
||||
consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := time.Now()
|
||||
hub := &oneHandler{}
|
||||
aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held },
|
||||
sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}}
|
||||
_ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again
|
||||
hub.handle(next("m5", "0.1.0"))
|
||||
state := versions.Reader{Keys: func() ([]string, error) {
|
||||
l, err := consumers.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var keys []string
|
||||
for k := range l.Keys() {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys, nil
|
||||
}, Get: func(k string) (json.RawMessage, bool, error) {
|
||||
e, err := consumers.Get(ctx, k)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value(), true, nil
|
||||
}}
|
||||
if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") {
|
||||
t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a)
|
||||
}
|
||||
if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy {
|
||||
t.Errorf("the health four minutes in = %+v, want healthy", a)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package versions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// BuilderHealth is builder_health's answer at now (design 47 H2 §794, H-A-435, H-A-462): healthy when each of the builder's
|
||||
// durable consumers' oldest unacknowledged message waits less than behindAfter, as the consumers' facts the builder writes at
|
||||
// every delivery and every acknowledgement say them (its state consumers, read through consumers); otherwise unhealthy, naming
|
||||
// each consumer behind and how long. The judgement is the module's shared code, so the builder's whole test proves it.
|
||||
func BuilderHealth(consumers Reader, now time.Time, behindAfter time.Duration) health.Answer {
|
||||
keys, err := consumers.Keys()
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err))
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState))
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var behind []string
|
||||
for _, k := range keys {
|
||||
raw, ok, err := consumers.Get(k)
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err))
|
||||
}
|
||||
var f process.ConsumerFacts
|
||||
if ok {
|
||||
if err := json.Unmarshal(raw, &f); err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err))
|
||||
}
|
||||
}
|
||||
if !f.OldestUnacknowledged.IsZero() {
|
||||
if age := now.Sub(f.OldestUnacknowledged); age >= behindAfter {
|
||||
behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", behindAfter, strings.Join(behind, ", ")))
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), behindAfter))
|
||||
}
|
||||
Reference in New Issue
Block a user