diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go new file mode 100644 index 000000000..9b28d4df9 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/health.go @@ -0,0 +1,70 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-builder/internal/versions" +) + +const ( + // HealthTool is builder_health, the builder's health as its manifest's health asks it (design 47 H2 §794, H-A-435). + HealthTool = "builder_health" + // BehindAfter is how long a durable consumer's oldest unacknowledged message may wait before the health names it: an + // events backlog, or a handler standing still on one message (§794). + BehindAfter = 5 * time.Minute +) + +// healthTool is builder_health: healthy when it answers, and each of the builder's durable consumers' oldest unacknowledged +// message waits less than BehindAfter, as the consumers' facts the builder writes at every delivery and every acknowledgement +// say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise unhealthy, naming each +// consumer behind and how long. +func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool { + return stdio.Tool{Name: HealthTool, + Description: "The builder's health: healthy when it answers and each of its durable consumers' oldest unacknowledged message waits less " + + "than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " + + "Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { return builderHealth(consumers, now()), nil }} +} + +// builderHealth is builder_health's answer at now. +func builderHealth(consumers versions.Reader, now time.Time) health.Answer { + keys, err := consumers.Keys() + if err != nil { + return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err)) + } + if len(keys) == 0 { + return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState)) + } + sort.Strings(keys) + var behind []string + for _, k := range keys { + raw, ok, err := consumers.Get(k) + if err != nil { + return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err)) + } + var f process.ConsumerFacts + if ok { + if err := json.Unmarshal(raw, &f); err != nil { + return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err)) + } + } + if !f.OldestUnacknowledged.IsZero() { + if age := now.Sub(f.OldestUnacknowledged); age >= BehindAfter { + behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending)) + } + } + } + if len(behind) > 0 { + return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", BehindAfter, strings.Join(behind, ", "))) + } + return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter)) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 5475bb16c..a2c0c6c14 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatdef" "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" @@ -447,6 +448,7 @@ func main() { }, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) tools = append(tools, waitsTools(wr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) tools = append(tools, unbuildableTool(unbuildablePage(stdio.State(build.UnbuildableState)))) // the plans no seat builds (§795) + tools = append(tools, healthTool(readerOf(stdio.State(process.ConsumersState)), time.Now)) // the builder's health (§794, H-A-435) tools = append(tools, toolchainsTools(stdio.State(buildrun.ToolchainsHeardState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) // The builder's expiring keys (state bounds): what each bounds and when it ends. tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription, diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go index 595ebf91f..3f2842f23 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go @@ -7,6 +7,8 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-builder/internal/versions" ) // TestAStateIsReadInOnePage: design 47 H H-A-292, H-A-293 — builder_publish_queue reads each of its own states in one page and @@ -41,4 +43,21 @@ func TestAStateIsReadInOnePage(t *testing.T) { if _, err := onePage(unsupported, nil); !errors.Is(err, errors.ErrUnsupported) { t.Errorf("a runtime with no pages answered %v, want ErrUnsupported so the reader reads key by key", err) } + // Design 47 H2 §794, H-A-435: builder_health reads the state consumers; a consumer whose oldest unacknowledged message waits + // six minutes is named, one within five minutes is not. + t.Run("builder_health names a consumer behind by five minutes", func(t *testing.T) { + now := time.Unix(10000, 0) + facts := map[string]json.RawMessage{ + "builder-events": json.RawMessage(`{"stream":"EVENTS","name":"builder-events","pending":40,"oldest-unacknowledged":"` + now.Add(-6*time.Minute).UTC().Format(time.RFC3339) + `"}`), + "builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)} + r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil }, + Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }} + a := builderHealth(r, now) + if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." { + t.Errorf("health = %+v, want unhealthy naming builder-events alone", a) + } + if a := builderHealth(r, now.Add(-2*time.Minute)); !a.Healthy { + t.Errorf("health four minutes in = %+v, want healthy", a) + } + }) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/consumers.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/consumers.go new file mode 100644 index 000000000..3d21bc606 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/consumers.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/process" +) + +// handlers is where a part of the builder registers its handler of the events it hears: the event hub, or agedHub over it. +type handlers interface { + Handle(pattern string, handle func(jetstream.Msg)) error +} + +// ledgers are the builder's durable consumers' ledgers, one per consumer as a message's metadata names it (the events consumer +// and each kept seat event's), writing each consumer's facts into the state consumers, which builder_health reads (design 47 H2 +// §794, H-A-435 and its addendum: the runner's runner_health form). +type ledgers struct { + sink process.ConsumerSink + logf func(string, ...any) + now func() time.Time + + mu sync.Mutex + byName map[string]*process.ConsumerLedger +} + +// of is the ledger of the consumer m came on, nil for a message that names none. +func (l *ledgers) of(m jetstream.Msg) *process.ConsumerLedger { + md, err := m.Metadata() + if err != nil || md == nil || md.Consumer == "" { + return nil + } + l.mu.Lock() + defer l.mu.Unlock() + if l.byName == nil { + l.byName = map[string]*process.ConsumerLedger{} + } + led, ok := l.byName[md.Consumer] + if !ok { + led = process.NewConsumerLedger(md.Stream, md.Consumer, l.sink, l.now, l.logf) + l.byName[md.Consumer] = led + } + return led +} + +// agedHub registers each handler on the hub so that every message it is handed is noted delivered on its consumer's ledger, +// and applied once the handler acknowledges or ends it (the runner's agedHub); a message the handler asks again stays +// unacknowledged, and grows old. +type agedHub struct { + hub handlers + ledgers *ledgers +} + +func (a agedHub) Handle(pattern string, handle func(jetstream.Msg)) error { + return a.hub.Handle(pattern, func(m jetstream.Msg) { + led := a.ledgers.of(m) + if led == nil { + handle(m) + return + } + led.Delivered(m) + handle(&agedMsg{Msg: m, ledger: led}) + }) +} + +// agedMsg is a message whose acknowledgement or end is noted applied on its consumer's ledger, once. +type agedMsg struct { + jetstream.Msg + ledger *process.ConsumerLedger + once sync.Once +} + +func (m *agedMsg) applied() { m.once.Do(func() { m.ledger.Applied(m.Msg, time.Time{}) }) } + +func (m *agedMsg) Ack() error { defer m.applied(); return m.Msg.Ack() } + +func (m *agedMsg) DoubleAck(ctx context.Context) error { + defer m.applied() + return m.Msg.DoubleAck(ctx) +} + +func (m *agedMsg) Term() error { defer m.applied(); return m.Msg.Term() } + +func (m *agedMsg) TermWithReason(r string) error { defer m.applied(); return m.Msg.TermWithReason(r) } + +// consumersSink writes a consumer's facts in the state consumers; while the state is not on the bus the bucket keeps the latest. +func consumersSink(b *process.AwaitedBucket) process.ConsumerSink { + return func(ctx context.Context, k string, v []byte) error { + _, err := b.Put(ctx, k, v) + if errors.Is(err, process.ErrNotYet) { + return nil + } + return err + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go new file mode 100644 index 000000000..5f073bc21 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/eventtake.go @@ -0,0 +1,39 @@ +package main + +import ( + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// SlowEvent is how long the handling of one heard event may take before the builder says it in its journal (design 47 H2 +// §794): the line names the event's subject, the part that took it and how long it took. +const SlowEvent = 100 * time.Millisecond + +// eventTaker is one part of the builder that takes events off its durable consumer: take answers whether the message was its. +type eventTaker struct { + part string + take func(jetstream.Msg) bool +} + +// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed SlowEvent +// (design 47 H2 §794). now is the clock, a test's when set. +func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() time.Time, logf func(string, ...any)) { + if now == nil { + now = time.Now + } + start := now() + part := rest.part + for _, t := range takers { + if t.take(m) { + part = t.part + break + } + } + if part == rest.part { + rest.take(m) + } + if took := now().Sub(start); took > SlowEvent { + logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), SlowEvent, part) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 0f3f2c632..a0b21a7e8 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -460,18 +460,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf} // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} - events := func(m jetstream.Msg) { - if decided.heard(m) { // the operator's answer to an ask about a key (§568) - return - } - if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568) - return - } - if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seatHeard.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) - return - } - heardGone(ctx, b, m, logger.Printf) + // Each part takes the events it hears in turn; a handling over SlowEvent is said with the part that took it (design 47 H2 §794). + takers := []eventTaker{ + {"the operator's answer to an ask about a key", decided.heard}, // §568 + {"the retake of a key failed on a transport fault", func(m jetstream.Msg) bool { return retake.heard(ctx, m, logger.Printf) }}, // §568 + {"what a node's engine reads", func(m jetstream.Msg) bool { return reads.heard(m, logger.Printf) }}, // manifest-reads-changed (ADR 0618 §4) + {"the toolchains reported", func(m jetstream.Msg) bool { return heardTools.Reported(ctx, m) }}, + {"the toolchains running", func(m jetstream.Msg) bool { return heardTools.Running(ctx, m) }}, + {"the floor", func(m jetstream.Msg) bool { return floor.heard(ctx, m) }}, + {"the reservations", func(m jetstream.Msg) bool { return reserved.heard(ctx, m) }}, + {"the registry seats' outcomes", func(m jetstream.Msg) bool { return seatHeard.heard(ctx, m) }}, } + gone := eventTaker{"the artifacts gone", func(m jetstream.Msg) bool { heardGone(ctx, b, m, logger.Printf); return true }} + events := func(m jetstream.Msg) { takeEvent(m, takers, gone, nil, logger.Printf) } // The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, the indexer's take-in.taken and // manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the // catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has @@ -511,10 +512,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } // A change of the seat list judges every plan no seat builds again and says them in one line (design 47 H2 §795). - seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx); b.SayUnbuildable(ctx) }, logger.Printf) + // Every message the builder's own handlers take is noted on its consumer's ledger, in the state consumers that + // builder_health reads (design 47 H2 §794, H-A-435). + aged := agedHub{hub: hub, ledgers: &ledgers{logf: logger.Printf, sink: consumersSink(process.AwaitBucket(ctx, js, conn, + builds.StateBucket(cred.Module, process.ConsumersState), "builder_health reads no consumer's age", logger.Printf))}} + seatGate.Follow(seatlist.ProcessBus(calls.Call, aged), func() { seatsChanged(ctx); b.SayUnbuildable(ctx) }, logger.Printf) half.holders = func() { seatsChanged(ctx) } for _, pattern := range seatEventPatterns { - if err := hub.Handle(pattern, events); err != nil { + if err := aged.Handle(pattern, events); err != nil { return err } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 94058086e..daad767b1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -531,4 +531,55 @@ func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) { if store.reads != 2 || !strings.Contains(strings.Join(logs, "\n"), "m05 holds 1 built digests") || !strings.Contains(strings.Join(logs, "\n"), "14 reservations stand at it") { t.Errorf("%d reads for two publisheds, journal %q: want one read each and the line made from it", store.reads, logs) } + // Design 47 H2 §794: an event whose handling passed 100 ms is said with how long and the part that took it; one within it + // is not. And H-A-435: every message the builder's handlers take is noted on its consumer's ledger, so the state consumers + // that builder_health reads holds a message left unacknowledged until it is acknowledged. + t.Run("a slow handling is said and a message left unacknowledged is kept", func(t *testing.T) { + var said []string + clock := time.Unix(1000, 0) + now := func() time.Time { return clock } + slow := []eventTaker{{"the reservations", func(jetstream.Msg) bool { clock = clock.Add(150 * time.Millisecond); return true }}} + fast := []eventTaker{{"the floor", func(jetstream.Msg) bool { clock = clock.Add(100 * time.Millisecond); return true }}} + rest := eventTaker{"the artifacts gone", func(jetstream.Msg) bool { return true }} + takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) + takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) + if len(said) != 1 || said[0] != "the event a.published took 150ms to handle, over 100ms: the reservations took it" { + t.Errorf("said %q, want one line for the handling over 100 ms", said) + } + facts := map[string]string{} + l := &ledgers{now: now, sink: func(_ context.Context, k string, v []byte) error { facts[k] = string(v); return nil }} + hub := &oneHandler{} + aged := agedHub{hub: hub, ledgers: l} + _ = aged.Handle("x.>", func(m jetstream.Msg) { + if m.Subject() == "x.ack" { + _ = m.Ack() + } + }) + hub.handle(&agedTestMsg{subject: "x.left", seq: 7}) + if !strings.Contains(facts["builder-events"], `"oldest-unacknowledged":"1970-01-01T00:16:40.25Z"`) { + t.Errorf("facts %q, want the message left unacknowledged kept with its delivery", facts) + } + hub.handle(&agedTestMsg{subject: "x.ack", seq: 8}) + if !strings.Contains(facts["builder-events"], `"acknowledged":8`) { + t.Errorf("facts %q, want the acknowledged message applied", facts) + } + }) +} + +// oneHandler is an event hub with one handler. +type oneHandler struct{ handle func(jetstream.Msg) } + +func (h *oneHandler) Handle(_ string, f func(jetstream.Msg)) error { h.handle = f; return nil } + +// agedTestMsg is a message of the builder's events consumer, with the metadata a ledger reads. +type agedTestMsg struct { + jetstream.Msg + subject string + seq uint64 +} + +func (m *agedTestMsg) Subject() string { return m.subject } +func (m *agedTestMsg) Ack() error { return nil } +func (m *agedTestMsg) Metadata() (*jetstream.MsgMetadata, error) { + return &jetstream.MsgMetadata{Sequence: jetstream.SequencePair{Stream: m.seq}, NumDelivered: 1, Consumer: "builder-events", Stream: "EVENTS"}, nil } diff --git a/modules/mesh-delivery-builder/internal/versions/index_test.go b/modules/mesh-delivery-builder/internal/versions/index_test.go index d946002aa..b7f86ad50 100644 --- a/modules/mesh-delivery-builder/internal/versions/index_test.go +++ b/modules/mesh-delivery-builder/internal/versions/index_test.go @@ -14,6 +14,13 @@ import ( type countingReserve struct { *reserveMem lists, reads atomic.Int64 + // indexReads and writes count the reads of an index key and the writes of any key (design 47 H2 §794). + indexReads, writes atomic.Int64 +} + +func (c *countingReserve) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) { + c.writes.Add(1) + return c.reserveMem.Update(ctx, key, value, rev) } func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string, error) { @@ -24,6 +31,8 @@ func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string, func (c *countingReserve) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { if !strings.HasPrefix(key, "index.") { c.reads.Add(1) + } else { + c.indexReads.Add(1) } return c.reserveMem.Get(ctx, key) } @@ -52,6 +61,31 @@ func TestTheIndexFindsAPublishedReservationInOneRead(t *testing.T) { store.reads.Store(0) return r, store } + // Design 47 H2 §794: 1,000 published events over 2,000 reservations cost one read of the index and one write each. + t.Run("1,000 publisheds over 2,000 reservations read the index once and write once each", func(t *testing.T) { + store := &countingReserve{reserveMem: &reserveMem{mem: newMem()}} + r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }} + for i := range 2000 { + m := fmt.Sprintf("r%04d", i) + if _, err := r.Reserve(ctx, m, "0.1", []string{m + ":a"}, Source{Fingerprint: "fp" + m}); err != nil { + t.Fatal(err) + } + if _, err := r.Built(ctx, m, "0.1.0", m+":a", "sha256:"+m, nil); err != nil { + t.Fatal(err) + } + } + store.indexReads.Store(0) + store.writes.Store(0) + for i := range 1000 { + m := fmt.Sprintf("r%04d", 2*i) + if marked, _, err := r.Published(ctx, m+"-a", "0.1.0", "sha256:"+m); err != nil || len(marked) != 1 { + t.Fatalf("the published of %s marked %v (%v)", m, marked, err) + } + } + if ir, w, l := store.indexReads.Load(), store.writes.Load(), store.lists.Load(); ir != 1000 || w != 1000 || l != 0 { + t.Errorf("1,000 publisheds read the index %d times, wrote %d and listed %d, want 1,000 reads, 1,000 writes and no listing", ir, w, l) + } + }) t.Run("a published by digest reads its one reservation", func(t *testing.T) { r, store := fill(t) marked, _, err := r.Published(ctx, "m10-a", "v0.1.0", "sha256:m10") diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index e2deaff68..33b9a37e0 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -115,7 +115,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}, {Name: "unbuildable", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -177,7 +177,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. if want := []string{"builder_announce_again", "builder_ask_again", "builder_ask_drop", "builder_asks", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", - "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_unbuildable", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) { + "builder_floor_again", "builder_floor_waits", "builder_health", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_unbuildable", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes. @@ -244,6 +244,10 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-builder"}) { t.Errorf("process = %v", p) } + // Design 47 H2 §794, H-A-435: the health asks builder_health, which names a consumer behind by five minutes. + if h, _ := p["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "builder_health" { + t.Errorf("health = %v, want the tool builder_health", p["health"]) + } env, _ := p["env"].(map[string]any) if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"]) diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index b7ef33184..5ed620a68 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -38,6 +38,7 @@ "builder_toolchains_drop", "builder_waits", "builder_unbuildable", + "builder_health", "builder_ask_drop", "builder_asks", "builder_versions", @@ -142,7 +143,8 @@ { "name": "reasks", "history": 1 }, { "name": "versions", "history": 1 }, { "name": "first-views", "history": 1 }, - { "name": "unbuildable", "history": 1, "max-mib": 8 } + { "name": "unbuildable", "history": 1, "max-mib": 8 }, + { "name": "consumers", "history": 1, "max-mib": 8 } ], "streams": [ { "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 }, @@ -344,7 +346,8 @@ }, "user": "mesh-delivery-builder", "health": { - "kind": "unit" + "kind": "tool", + "tool": "builder_health" } } ],