diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go index 9b28d4df9..f4093e846 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go @@ -1,15 +1,9 @@ package main import ( - "encoding/json" - "fmt" - "sort" - "strings" "time" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/health" - "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-builder/internal/versions" ) @@ -32,39 +26,5 @@ func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool { "than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " + "Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return builderHealth(consumers, now()), nil }} -} - -// builderHealth is builder_health's answer at now. -func builderHealth(consumers versions.Reader, now time.Time) health.Answer { - keys, err := consumers.Keys() - if err != nil { - return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err)) - } - if len(keys) == 0 { - return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState)) - } - sort.Strings(keys) - var behind []string - for _, k := range keys { - raw, ok, err := consumers.Get(k) - if err != nil { - return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err)) - } - var f process.ConsumerFacts - if ok { - if err := json.Unmarshal(raw, &f); err != nil { - return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err)) - } - } - if !f.OldestUnacknowledged.IsZero() { - if age := now.Sub(f.OldestUnacknowledged); age >= BehindAfter { - behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending)) - } - } - } - if len(behind) > 0 { - return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", BehindAfter, strings.Join(behind, ", "))) - } - return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter)) + Run: func(map[string]any) (any, error) { return versions.BuilderHealth(consumers, now(), BehindAfter), nil }} } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go index 49c5e349c..d82c08208 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go @@ -58,11 +58,11 @@ func TestAStateIsReadInOnePage(t *testing.T) { "builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)} r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil }, Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }} - a := builderHealth(r, now) + a := versions.BuilderHealth(r, now, BehindAfter) if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." { t.Errorf("health = %+v, want unhealthy naming builder-events alone", a) } - if a := builderHealth(r, now.Add(-2*time.Minute)); !a.Healthy { + if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), BehindAfter); !a.Healthy { t.Errorf("health four minutes in = %+v, want healthy", a) } }) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index f3459e66f..0ea8fa90f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -8,24 +8,30 @@ import ( "errors" "fmt" "slices" + "strings" "sync" "testing" "time" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-builder/internal/versions" ) // Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder. // A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and -// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and asks the -// bus module to drop the old state. The seat mesh-builds' verb publish-queue reads the stream: counts per holder and class, the -// oldest and newest, the live item listed first (H-A-454). +// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the +// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts +// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit and +// §794's backlog held six minutes, as steps (H-A-453 (3)). func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { b := startBed(t) ctx := b.ctx @@ -132,11 +138,20 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { if err != nil { t.Fatal(err) } - call := func(string, any) (json.RawMessage, error) { + var tagsWritten []string + call := func(tool string, body any) (json.RawMessage, error) { + switch tool { + case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796) + return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}}) + case "seat:" + gitseat.Seat + ".tag": + tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit) + return json.Marshal(gitseat.TagAnswer{Performed: true}) + } return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) } + found := &memFindings{vals: map[string][]byte{}} rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, - &findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf) + &findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf) rv.filled.Store(true) type oldItem struct { key string @@ -196,4 +211,84 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 { t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects) } + + // Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another + // commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags /v at + // the build's commit, and the module's version-released clears the finding. + rv.r.Names = func(context.Context, string, string) (string, bool, error) { + return "git.example/o/r/modules/m4", true, nil + } + v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib", + Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1}) + elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere) + if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised { + t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised) + } + catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events + if err != nil { + t.Fatal(err) + } + evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy, + FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy}) + if err != nil { + t.Fatal(err) + } + next := func(module, version string) jetstream.Msg { + t.Helper() + data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version}) + if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil { + t.Fatal(err) + } + m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second)) + if err != nil { + t.Fatal(err) + } + return m + } + if !rv.heard(ctx, next("m4", "0.2.1")) { + t.Fatal("the reservations did not take version-released") + } + if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) { + t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten) + } + + // Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its + // consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five + // minutes it does not. + consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)}) + if err != nil { + t.Fatal(err) + } + held := time.Now() + hub := &oneHandler{} + aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held }, + sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}} + _ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again + hub.handle(next("m5", "0.1.0")) + state := versions.Reader{Keys: func() ([]string, error) { + l, err := consumers.ListKeys(ctx) + if err != nil { + return nil, err + } + var keys []string + for k := range l.Keys() { + keys = append(keys, k) + } + return keys, nil + }, Get: func(k string) (json.RawMessage, bool, error) { + e, err := consumers.Get(ctx, k) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil + }} + if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") { + t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a) + } + if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy { + t.Errorf("the health four minutes in = %+v, want healthy", a) + } } diff --git a/modules/mesh-delivery-builder/internal/versions/health.go b/modules/mesh-delivery-builder/internal/versions/health.go new file mode 100644 index 000000000..22d2c9193 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/health.go @@ -0,0 +1,49 @@ +package versions + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// BuilderHealth is builder_health's answer at now (design 47 H2 §794, H-A-435, H-A-462): healthy when each of the builder's +// durable consumers' oldest unacknowledged message waits less than behindAfter, as the consumers' facts the builder writes at +// every delivery and every acknowledgement say them (its state consumers, read through consumers); otherwise unhealthy, naming +// each consumer behind and how long. The judgement is the module's shared code, so the builder's whole test proves it. +func BuilderHealth(consumers Reader, now time.Time, behindAfter time.Duration) health.Answer { + keys, err := consumers.Keys() + if err != nil { + return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err)) + } + if len(keys) == 0 { + return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState)) + } + sort.Strings(keys) + var behind []string + for _, k := range keys { + raw, ok, err := consumers.Get(k) + if err != nil { + return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err)) + } + var f process.ConsumerFacts + if ok { + if err := json.Unmarshal(raw, &f); err != nil { + return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err)) + } + } + if !f.OldestUnacknowledged.IsZero() { + if age := now.Sub(f.OldestUnacknowledged); age >= behindAfter { + behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending)) + } + } + } + if len(behind) > 0 { + return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", behindAfter, strings.Join(behind, ", "))) + } + return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), behindAfter)) +}