diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go index f4093e846..75768393d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go @@ -4,6 +4,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" "mesh-delivery-builder/internal/versions" ) @@ -11,20 +12,29 @@ import ( const ( // HealthTool is builder_health, the builder's health as its manifest's health asks it (design 47 H2 §794, H-A-435). HealthTool = "builder_health" - // BehindAfter is how long a durable consumer's oldest unacknowledged message may wait before the health names it: an - // events backlog, or a handler standing still on one message (§794). - BehindAfter = 5 * time.Minute + // eventsBehindBoundSetting is the setting events-behind-bound (§794; H-A-461 (2), P11): how long a durable consumer's + // oldest unacknowledged message may wait before the health names it, an events backlog or a handler standing still on one + // message; defaultEventsBehindBound when none is set. + eventsBehindBoundSetting = "events-behind-bound" + defaultEventsBehindBound = 5 * time.Minute ) // healthTool is builder_health: healthy when it answers, and each of the builder's durable consumers' oldest unacknowledged -// message waits less than BehindAfter, as the consumers' facts the builder writes at every delivery and every acknowledgement -// say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise unhealthy, naming each -// consumer behind and how long. +// message waits less than the setting events-behind-bound, as the consumers' facts the builder writes at every delivery and +// every acknowledgement say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise +// unhealthy, naming each consumer behind and how long. A setting that holds no duration is said as the health's answer. func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool { return stdio.Tool{Name: HealthTool, Description: "The builder's health: healthy when it answers and each of its durable consumers' oldest unacknowledged message waits less " + - "than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " + - "Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.", + "than the setting events-behind-bound (five minutes by default; no events backlog, no handler standing still on a message); otherwise it " + + "names each consumer behind and how long. Read from the builder's state consumers, which it writes at every message delivered and " + + "acknowledged. Read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return versions.BuilderHealth(consumers, now(), BehindAfter), nil }} + Run: func(map[string]any) (any, error) { + bound, err := stdio.SettingDuration(eventsBehindBoundSetting, defaultEventsBehindBound) + if err != nil { + return health.Unhealthy("The builder's " + err.Error() + "."), nil + } + return versions.BuilderHealth(consumers, now(), bound), nil + }} } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go index d82c08208..f02bd22ca 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" "mesh-delivery-builder/internal/build" @@ -58,13 +59,23 @@ func TestAStateIsReadInOnePage(t *testing.T) { "builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)} r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil }, Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }} - a := versions.BuilderHealth(r, now, BehindAfter) + a := versions.BuilderHealth(r, now, defaultEventsBehindBound) if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." { t.Errorf("health = %+v, want unhealthy naming builder-events alone", a) } - if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), BehindAfter); !a.Healthy { + if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), defaultEventsBehindBound); !a.Healthy { t.Errorf("health four minutes in = %+v, want healthy", a) } + // H-A-461 (2): the bound is the setting events-behind-bound, five minutes when none is set. + tool := healthTool(r, func() time.Time { return now }) + for _, c := range []struct{ set, want string }{{"", "unhealthy"}, {"10m", "healthy"}, {"soon", "events-behind-bound"}} { + t.Setenv(stdio.SettingWord(eventsBehindBoundSetting), c.set) + got, _ := tool.Run(nil) + a := got.(health.Answer) + if (c.want == "healthy") != a.Healthy || (c.want == "events-behind-bound" && !strings.Contains(a.Why, c.want)) { + t.Errorf("with events-behind-bound %q the health = %+v, want %s", c.set, a, c.want) + } + } }) // Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from // its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go index 5f073bc21..ff7d41ed3 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go @@ -1,14 +1,30 @@ package main import ( + "errors" "time" "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" ) -// SlowEvent is how long the handling of one heard event may take before the builder says it in its journal (design 47 H2 -// §794): the line names the event's subject, the part that took it and how long it took. -const SlowEvent = 100 * time.Millisecond +// slowEventSetting is the setting slow-event (design 47 H2 §794; H-A-461 (2), P11): how long the handling of one heard event +// may take before the builder says it in its journal, the line naming the event's subject, the part that took it and how long +// it took; defaultSlowEvent when none is set. +const ( + slowEventSetting = "slow-event" + defaultSlowEvent = 100 * time.Millisecond +) + +// slowEvent reads the setting slow-event. +func slowEvent() (time.Duration, error) { + d, err := stdio.SettingDuration(slowEventSetting, defaultSlowEvent) + if err != nil { + return 0, errors.New("the builder's " + err.Error()) + } + return d, nil +} // eventTaker is one part of the builder that takes events off its durable consumer: take answers whether the message was its. type eventTaker struct { @@ -16,9 +32,9 @@ type eventTaker struct { take func(jetstream.Msg) bool } -// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed SlowEvent -// (design 47 H2 §794). now is the clock, a test's when set. -func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() time.Time, logf func(string, ...any)) { +// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed slow, the +// setting slow-event (design 47 H2 §794). now is the clock, a test's when set. +func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, slow time.Duration, now func() time.Time, logf func(string, ...any)) { if now == nil { now = time.Now } @@ -33,7 +49,7 @@ func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() if part == rest.part { rest.take(m) } - if took := now().Sub(start); took > SlowEvent { - logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), SlowEvent, part) + if took := now().Sub(start); took > slow { + logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), slow, part) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index b6a6934a6..faf0ad6c0 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -479,7 +479,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf} // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} - // Each part takes the events it hears in turn; a handling over SlowEvent is said with the part that took it (design 47 H2 §794). + // Each part takes the events it hears in turn; a handling over the setting slow-event is said with the part that took it + // (design 47 H2 §794; H-A-461 (2)). + slow, err := slowEvent() + if err != nil { + return err + } takers := []eventTaker{ {"the operator's answer to an ask about a key", decided.heard}, // §568 {"the retake of a key failed on a transport fault", func(m jetstream.Msg) bool { return retake.heard(ctx, m, logger.Printf) }}, // §568 @@ -491,7 +496,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro {"the registry seats' outcomes", func(m jetstream.Msg) bool { return seatHeard.heard(ctx, m) }}, } gone := eventTaker{"the artifacts gone", func(m jetstream.Msg) bool { heardGone(ctx, b, m, logger.Printf); return true }} - events := func(m jetstream.Msg) { takeEvent(m, takers, gone, nil, logger.Printf) } + events := func(m jetstream.Msg) { takeEvent(m, takers, gone, slow, nil, logger.Printf) } // The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, the indexer's take-in.taken and // manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the // catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 6d744c14d..874e41f9f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -16,6 +16,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" @@ -587,11 +588,27 @@ func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) { slow := []eventTaker{{"the reservations", func(jetstream.Msg) bool { clock = clock.Add(150 * time.Millisecond); return true }}} fast := []eventTaker{{"the floor", func(jetstream.Msg) bool { clock = clock.Add(100 * time.Millisecond); return true }}} rest := eventTaker{"the artifacts gone", func(jetstream.Msg) bool { return true }} - takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) - takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) + takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) + takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) if len(said) != 1 || said[0] != "the event a.published took 150ms to handle, over 100ms: the reservations took it" { t.Errorf("said %q, want one line for the handling over 100 ms", said) } + // H-A-461 (2): the bound is the setting slow-event, 100 ms when none is set; at 200ms the 150 ms handling is not said. + for _, c := range []struct { + set string + want time.Duration + }{{"", defaultSlowEvent}, {"200ms", 200 * time.Millisecond}, {"soon", 0}} { + t.Setenv(stdio.SettingWord(slowEventSetting), c.set) + if d, err := slowEvent(); d != c.want || (c.want == 0) != (err != nil) { + t.Errorf("with slow-event %q the bound is %v (%v), want %v", c.set, d, err, c.want) + } + } + said = nil + takeEvent(&agedTestMsg{subject: "a.published", seq: 3}, slow, rest, 200*time.Millisecond, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) + if len(said) != 0 { + t.Errorf("said %q with slow-event 200ms, want nothing for a 150 ms handling", said) + } + clock = clock.Add(-150 * time.Millisecond) // the ledger's times below read as before this row facts := map[string]string{} l := &ledgers{now: now, sink: func(_ context.Context, k string, v []byte) error { facts[k] = string(v); return nil }} hub := &oneHandler{} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index e2dde9f47..5e552a268 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -200,10 +200,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 { t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams) } - if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 10 || + if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 || // design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" || - !m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) || + !m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || + // H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default + m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime || + m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) || m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 { t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 7fc4435e5..ffbd8b463 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -204,6 +204,20 @@ "default": "30s", "runtime": true }, + "slow-event": { + "type": "string", + "description": "how long the handling of one heard event may take before the builder says it in its journal, with its duration and the part that took it, a Go duration (design 47 H2 §794; H-A-461 (2)).", + "trusted": false, + "default": "100ms", + "runtime": true + }, + "events-behind-bound": { + "type": "string", + "description": "how long a durable consumer's oldest unacknowledged message may wait before builder_health names it, an events backlog or a handler standing still, a Go duration (design 47 H2 §794; H-A-461 (2)).", + "trusted": false, + "default": "5m", + "runtime": true + }, "publish-pause": { "type": "number", "description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).",