builder: the slow-event and events-behind-bound bounds are settings (F3)
The review found §794's two bounds fixed in code: the builder's 100 ms for saying a slow event's handling, and builder_health's five minutes for naming a consumer behind. Both become settings read at run time, slow-event (100ms) and events-behind-bound (5m), declared in the manifest's config with those defaults; a value that is no duration refuses the builder's start, and makes the health say it. Builds: design 47 H2 §794, H-A-461 (2)
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
@@ -11,20 +12,29 @@ import (
|
||||
const (
|
||||
// HealthTool is builder_health, the builder's health as its manifest's health asks it (design 47 H2 §794, H-A-435).
|
||||
HealthTool = "builder_health"
|
||||
// BehindAfter is how long a durable consumer's oldest unacknowledged message may wait before the health names it: an
|
||||
// events backlog, or a handler standing still on one message (§794).
|
||||
BehindAfter = 5 * time.Minute
|
||||
// eventsBehindBoundSetting is the setting events-behind-bound (§794; H-A-461 (2), P11): how long a durable consumer's
|
||||
// oldest unacknowledged message may wait before the health names it, an events backlog or a handler standing still on one
|
||||
// message; defaultEventsBehindBound when none is set.
|
||||
eventsBehindBoundSetting = "events-behind-bound"
|
||||
defaultEventsBehindBound = 5 * time.Minute
|
||||
)
|
||||
|
||||
// healthTool is builder_health: healthy when it answers, and each of the builder's durable consumers' oldest unacknowledged
|
||||
// message waits less than BehindAfter, as the consumers' facts the builder writes at every delivery and every acknowledgement
|
||||
// say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise unhealthy, naming each
|
||||
// consumer behind and how long.
|
||||
// message waits less than the setting events-behind-bound, as the consumers' facts the builder writes at every delivery and
|
||||
// every acknowledgement say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise
|
||||
// unhealthy, naming each consumer behind and how long. A setting that holds no duration is said as the health's answer.
|
||||
func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool {
|
||||
return stdio.Tool{Name: HealthTool,
|
||||
Description: "The builder's health: healthy when it answers and each of its durable consumers' oldest unacknowledged message waits less " +
|
||||
"than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " +
|
||||
"Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.",
|
||||
"than the setting events-behind-bound (five minutes by default; no events backlog, no handler standing still on a message); otherwise it " +
|
||||
"names each consumer behind and how long. Read from the builder's state consumers, which it writes at every message delivered and " +
|
||||
"acknowledged. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return versions.BuilderHealth(consumers, now(), BehindAfter), nil }}
|
||||
Run: func(map[string]any) (any, error) {
|
||||
bound, err := stdio.SettingDuration(eventsBehindBoundSetting, defaultEventsBehindBound)
|
||||
if err != nil {
|
||||
return health.Unhealthy("The builder's " + err.Error() + "."), nil
|
||||
}
|
||||
return versions.BuilderHealth(consumers, now(), bound), nil
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
|
||||
@@ -58,13 +59,23 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
"builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)}
|
||||
r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil },
|
||||
Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }}
|
||||
a := versions.BuilderHealth(r, now, BehindAfter)
|
||||
a := versions.BuilderHealth(r, now, defaultEventsBehindBound)
|
||||
if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." {
|
||||
t.Errorf("health = %+v, want unhealthy naming builder-events alone", a)
|
||||
}
|
||||
if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), BehindAfter); !a.Healthy {
|
||||
if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), defaultEventsBehindBound); !a.Healthy {
|
||||
t.Errorf("health four minutes in = %+v, want healthy", a)
|
||||
}
|
||||
// H-A-461 (2): the bound is the setting events-behind-bound, five minutes when none is set.
|
||||
tool := healthTool(r, func() time.Time { return now })
|
||||
for _, c := range []struct{ set, want string }{{"", "unhealthy"}, {"10m", "healthy"}, {"soon", "events-behind-bound"}} {
|
||||
t.Setenv(stdio.SettingWord(eventsBehindBoundSetting), c.set)
|
||||
got, _ := tool.Run(nil)
|
||||
a := got.(health.Answer)
|
||||
if (c.want == "healthy") != a.Healthy || (c.want == "events-behind-bound" && !strings.Contains(a.Why, c.want)) {
|
||||
t.Errorf("with events-behind-bound %q the health = %+v, want %s", c.set, a, c.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
// Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from
|
||||
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and
|
||||
|
||||
@@ -1,14 +1,30 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// SlowEvent is how long the handling of one heard event may take before the builder says it in its journal (design 47 H2
|
||||
// §794): the line names the event's subject, the part that took it and how long it took.
|
||||
const SlowEvent = 100 * time.Millisecond
|
||||
// slowEventSetting is the setting slow-event (design 47 H2 §794; H-A-461 (2), P11): how long the handling of one heard event
|
||||
// may take before the builder says it in its journal, the line naming the event's subject, the part that took it and how long
|
||||
// it took; defaultSlowEvent when none is set.
|
||||
const (
|
||||
slowEventSetting = "slow-event"
|
||||
defaultSlowEvent = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
// slowEvent reads the setting slow-event.
|
||||
func slowEvent() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(slowEventSetting, defaultSlowEvent)
|
||||
if err != nil {
|
||||
return 0, errors.New("the builder's " + err.Error())
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// eventTaker is one part of the builder that takes events off its durable consumer: take answers whether the message was its.
|
||||
type eventTaker struct {
|
||||
@@ -16,9 +32,9 @@ type eventTaker struct {
|
||||
take func(jetstream.Msg) bool
|
||||
}
|
||||
|
||||
// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed SlowEvent
|
||||
// (design 47 H2 §794). now is the clock, a test's when set.
|
||||
func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() time.Time, logf func(string, ...any)) {
|
||||
// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed slow, the
|
||||
// setting slow-event (design 47 H2 §794). now is the clock, a test's when set.
|
||||
func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, slow time.Duration, now func() time.Time, logf func(string, ...any)) {
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
@@ -33,7 +49,7 @@ func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func()
|
||||
if part == rest.part {
|
||||
rest.take(m)
|
||||
}
|
||||
if took := now().Sub(start); took > SlowEvent {
|
||||
logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), SlowEvent, part)
|
||||
if took := now().Sub(start); took > slow {
|
||||
logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), slow, part)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -479,7 +479,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf}
|
||||
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
|
||||
seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
|
||||
// Each part takes the events it hears in turn; a handling over SlowEvent is said with the part that took it (design 47 H2 §794).
|
||||
// Each part takes the events it hears in turn; a handling over the setting slow-event is said with the part that took it
|
||||
// (design 47 H2 §794; H-A-461 (2)).
|
||||
slow, err := slowEvent()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
takers := []eventTaker{
|
||||
{"the operator's answer to an ask about a key", decided.heard}, // §568
|
||||
{"the retake of a key failed on a transport fault", func(m jetstream.Msg) bool { return retake.heard(ctx, m, logger.Printf) }}, // §568
|
||||
@@ -491,7 +496,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
{"the registry seats' outcomes", func(m jetstream.Msg) bool { return seatHeard.heard(ctx, m) }},
|
||||
}
|
||||
gone := eventTaker{"the artifacts gone", func(m jetstream.Msg) bool { heardGone(ctx, b, m, logger.Printf); return true }}
|
||||
events := func(m jetstream.Msg) { takeEvent(m, takers, gone, nil, logger.Printf) }
|
||||
events := func(m jetstream.Msg) { takeEvent(m, takers, gone, slow, nil, logger.Printf) }
|
||||
// The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, the indexer's take-in.taken and
|
||||
// manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the
|
||||
// catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
@@ -587,11 +588,27 @@ func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) {
|
||||
slow := []eventTaker{{"the reservations", func(jetstream.Msg) bool { clock = clock.Add(150 * time.Millisecond); return true }}}
|
||||
fast := []eventTaker{{"the floor", func(jetstream.Msg) bool { clock = clock.Add(100 * time.Millisecond); return true }}}
|
||||
rest := eventTaker{"the artifacts gone", func(jetstream.Msg) bool { return true }}
|
||||
takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
if len(said) != 1 || said[0] != "the event a.published took 150ms to handle, over 100ms: the reservations took it" {
|
||||
t.Errorf("said %q, want one line for the handling over 100 ms", said)
|
||||
}
|
||||
// H-A-461 (2): the bound is the setting slow-event, 100 ms when none is set; at 200ms the 150 ms handling is not said.
|
||||
for _, c := range []struct {
|
||||
set string
|
||||
want time.Duration
|
||||
}{{"", defaultSlowEvent}, {"200ms", 200 * time.Millisecond}, {"soon", 0}} {
|
||||
t.Setenv(stdio.SettingWord(slowEventSetting), c.set)
|
||||
if d, err := slowEvent(); d != c.want || (c.want == 0) != (err != nil) {
|
||||
t.Errorf("with slow-event %q the bound is %v (%v), want %v", c.set, d, err, c.want)
|
||||
}
|
||||
}
|
||||
said = nil
|
||||
takeEvent(&agedTestMsg{subject: "a.published", seq: 3}, slow, rest, 200*time.Millisecond, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
if len(said) != 0 {
|
||||
t.Errorf("said %q with slow-event 200ms, want nothing for a 150 ms handling", said)
|
||||
}
|
||||
clock = clock.Add(-150 * time.Millisecond) // the ledger's times below read as before this row
|
||||
facts := map[string]string{}
|
||||
l := &ledgers{now: now, sink: func(_ context.Context, k string, v []byte) error { facts[k] = string(v); return nil }}
|
||||
hub := &oneHandler{}
|
||||
|
||||
@@ -200,10 +200,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 {
|
||||
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
|
||||
}
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 10 ||
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 ||
|
||||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
|
||||
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
|
||||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime ||
|
||||
// H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default
|
||||
m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime ||
|
||||
m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 {
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
}
|
||||
|
||||
@@ -204,6 +204,20 @@
|
||||
"default": "30s",
|
||||
"runtime": true
|
||||
},
|
||||
"slow-event": {
|
||||
"type": "string",
|
||||
"description": "how long the handling of one heard event may take before the builder says it in its journal, with its duration and the part that took it, a Go duration (design 47 H2 §794; H-A-461 (2)).",
|
||||
"trusted": false,
|
||||
"default": "100ms",
|
||||
"runtime": true
|
||||
},
|
||||
"events-behind-bound": {
|
||||
"type": "string",
|
||||
"description": "how long a durable consumer's oldest unacknowledged message may wait before builder_health names it, an events backlog or a handler standing still, a Go duration (design 47 H2 §794; H-A-461 (2)).",
|
||||
"trusted": false,
|
||||
"default": "5m",
|
||||
"runtime": true
|
||||
},
|
||||
"publish-pause": {
|
||||
"type": "number",
|
||||
"description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).",
|
||||
|
||||
Reference in New Issue
Block a user