B5 (H-A-435): the builder's health is builder_health, and a slow event is one journal line
The builder said nothing when its events backed up or one handling stood still; its health now reads each durable consumer's oldest unacknowledged message from its state consumers, and an event handled over 100 ms is said with the part that took it. Builds: design 47 H2 §794, §820; H-A-435 and its addendum
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
const (
|
||||
// HealthTool is builder_health, the builder's health as its manifest's health asks it (design 47 H2 §794, H-A-435).
|
||||
HealthTool = "builder_health"
|
||||
// BehindAfter is how long a durable consumer's oldest unacknowledged message may wait before the health names it: an
|
||||
// events backlog, or a handler standing still on one message (§794).
|
||||
BehindAfter = 5 * time.Minute
|
||||
)
|
||||
|
||||
// healthTool is builder_health: healthy when it answers, and each of the builder's durable consumers' oldest unacknowledged
|
||||
// message waits less than BehindAfter, as the consumers' facts the builder writes at every delivery and every acknowledgement
|
||||
// say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise unhealthy, naming each
|
||||
// consumer behind and how long.
|
||||
func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool {
|
||||
return stdio.Tool{Name: HealthTool,
|
||||
Description: "The builder's health: healthy when it answers and each of its durable consumers' oldest unacknowledged message waits less " +
|
||||
"than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " +
|
||||
"Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return builderHealth(consumers, now()), nil }}
|
||||
}
|
||||
|
||||
// builderHealth is builder_health's answer at now.
|
||||
func builderHealth(consumers versions.Reader, now time.Time) health.Answer {
|
||||
keys, err := consumers.Keys()
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err))
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState))
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var behind []string
|
||||
for _, k := range keys {
|
||||
raw, ok, err := consumers.Get(k)
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err))
|
||||
}
|
||||
var f process.ConsumerFacts
|
||||
if ok {
|
||||
if err := json.Unmarshal(raw, &f); err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err))
|
||||
}
|
||||
}
|
||||
if !f.OldestUnacknowledged.IsZero() {
|
||||
if age := now.Sub(f.OldestUnacknowledged); age >= BehindAfter {
|
||||
behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", BehindAfter, strings.Join(behind, ", ")))
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter))
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
@@ -447,6 +448,7 @@ func main() {
|
||||
}, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
tools = append(tools, waitsTools(wr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
tools = append(tools, unbuildableTool(unbuildablePage(stdio.State(build.UnbuildableState)))) // the plans no seat builds (§795)
|
||||
tools = append(tools, healthTool(readerOf(stdio.State(process.ConsumersState)), time.Now)) // the builder's health (§794, H-A-435)
|
||||
tools = append(tools, toolchainsTools(stdio.State(buildrun.ToolchainsHeardState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
|
||||
// The builder's expiring keys (state bounds): what each bounds and when it ends.
|
||||
tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription,
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// TestAStateIsReadInOnePage: design 47 H H-A-292, H-A-293 — builder_publish_queue reads each of its own states in one page and
|
||||
@@ -41,4 +43,21 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
if _, err := onePage(unsupported, nil); !errors.Is(err, errors.ErrUnsupported) {
|
||||
t.Errorf("a runtime with no pages answered %v, want ErrUnsupported so the reader reads key by key", err)
|
||||
}
|
||||
// Design 47 H2 §794, H-A-435: builder_health reads the state consumers; a consumer whose oldest unacknowledged message waits
|
||||
// six minutes is named, one within five minutes is not.
|
||||
t.Run("builder_health names a consumer behind by five minutes", func(t *testing.T) {
|
||||
now := time.Unix(10000, 0)
|
||||
facts := map[string]json.RawMessage{
|
||||
"builder-events": json.RawMessage(`{"stream":"EVENTS","name":"builder-events","pending":40,"oldest-unacknowledged":"` + now.Add(-6*time.Minute).UTC().Format(time.RFC3339) + `"}`),
|
||||
"builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)}
|
||||
r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil },
|
||||
Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }}
|
||||
a := builderHealth(r, now)
|
||||
if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." {
|
||||
t.Errorf("health = %+v, want unhealthy naming builder-events alone", a)
|
||||
}
|
||||
if a := builderHealth(r, now.Add(-2*time.Minute)); !a.Healthy {
|
||||
t.Errorf("health four minutes in = %+v, want healthy", a)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// handlers is where a part of the builder registers its handler of the events it hears: the event hub, or agedHub over it.
|
||||
type handlers interface {
|
||||
Handle(pattern string, handle func(jetstream.Msg)) error
|
||||
}
|
||||
|
||||
// ledgers are the builder's durable consumers' ledgers, one per consumer as a message's metadata names it (the events consumer
|
||||
// and each kept seat event's), writing each consumer's facts into the state consumers, which builder_health reads (design 47 H2
|
||||
// §794, H-A-435 and its addendum: the runner's runner_health form).
|
||||
type ledgers struct {
|
||||
sink process.ConsumerSink
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
byName map[string]*process.ConsumerLedger
|
||||
}
|
||||
|
||||
// of is the ledger of the consumer m came on, nil for a message that names none.
|
||||
func (l *ledgers) of(m jetstream.Msg) *process.ConsumerLedger {
|
||||
md, err := m.Metadata()
|
||||
if err != nil || md == nil || md.Consumer == "" {
|
||||
return nil
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.byName == nil {
|
||||
l.byName = map[string]*process.ConsumerLedger{}
|
||||
}
|
||||
led, ok := l.byName[md.Consumer]
|
||||
if !ok {
|
||||
led = process.NewConsumerLedger(md.Stream, md.Consumer, l.sink, l.now, l.logf)
|
||||
l.byName[md.Consumer] = led
|
||||
}
|
||||
return led
|
||||
}
|
||||
|
||||
// agedHub registers each handler on the hub so that every message it is handed is noted delivered on its consumer's ledger,
|
||||
// and applied once the handler acknowledges or ends it (the runner's agedHub); a message the handler asks again stays
|
||||
// unacknowledged, and grows old.
|
||||
type agedHub struct {
|
||||
hub handlers
|
||||
ledgers *ledgers
|
||||
}
|
||||
|
||||
func (a agedHub) Handle(pattern string, handle func(jetstream.Msg)) error {
|
||||
return a.hub.Handle(pattern, func(m jetstream.Msg) {
|
||||
led := a.ledgers.of(m)
|
||||
if led == nil {
|
||||
handle(m)
|
||||
return
|
||||
}
|
||||
led.Delivered(m)
|
||||
handle(&agedMsg{Msg: m, ledger: led})
|
||||
})
|
||||
}
|
||||
|
||||
// agedMsg is a message whose acknowledgement or end is noted applied on its consumer's ledger, once.
|
||||
type agedMsg struct {
|
||||
jetstream.Msg
|
||||
ledger *process.ConsumerLedger
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (m *agedMsg) applied() { m.once.Do(func() { m.ledger.Applied(m.Msg, time.Time{}) }) }
|
||||
|
||||
func (m *agedMsg) Ack() error { defer m.applied(); return m.Msg.Ack() }
|
||||
|
||||
func (m *agedMsg) DoubleAck(ctx context.Context) error {
|
||||
defer m.applied()
|
||||
return m.Msg.DoubleAck(ctx)
|
||||
}
|
||||
|
||||
func (m *agedMsg) Term() error { defer m.applied(); return m.Msg.Term() }
|
||||
|
||||
func (m *agedMsg) TermWithReason(r string) error { defer m.applied(); return m.Msg.TermWithReason(r) }
|
||||
|
||||
// consumersSink writes a consumer's facts in the state consumers; while the state is not on the bus the bucket keeps the latest.
|
||||
func consumersSink(b *process.AwaitedBucket) process.ConsumerSink {
|
||||
return func(ctx context.Context, k string, v []byte) error {
|
||||
_, err := b.Put(ctx, k, v)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// SlowEvent is how long the handling of one heard event may take before the builder says it in its journal (design 47 H2
|
||||
// §794): the line names the event's subject, the part that took it and how long it took.
|
||||
const SlowEvent = 100 * time.Millisecond
|
||||
|
||||
// eventTaker is one part of the builder that takes events off its durable consumer: take answers whether the message was its.
|
||||
type eventTaker struct {
|
||||
part string
|
||||
take func(jetstream.Msg) bool
|
||||
}
|
||||
|
||||
// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed SlowEvent
|
||||
// (design 47 H2 §794). now is the clock, a test's when set.
|
||||
func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() time.Time, logf func(string, ...any)) {
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
start := now()
|
||||
part := rest.part
|
||||
for _, t := range takers {
|
||||
if t.take(m) {
|
||||
part = t.part
|
||||
break
|
||||
}
|
||||
}
|
||||
if part == rest.part {
|
||||
rest.take(m)
|
||||
}
|
||||
if took := now().Sub(start); took > SlowEvent {
|
||||
logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), SlowEvent, part)
|
||||
}
|
||||
}
|
||||
@@ -460,18 +460,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf}
|
||||
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
|
||||
seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
|
||||
events := func(m jetstream.Msg) {
|
||||
if decided.heard(m) { // the operator's answer to an ask about a key (§568)
|
||||
return
|
||||
}
|
||||
if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568)
|
||||
return
|
||||
}
|
||||
if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seatHeard.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4)
|
||||
return
|
||||
}
|
||||
heardGone(ctx, b, m, logger.Printf)
|
||||
// Each part takes the events it hears in turn; a handling over SlowEvent is said with the part that took it (design 47 H2 §794).
|
||||
takers := []eventTaker{
|
||||
{"the operator's answer to an ask about a key", decided.heard}, // §568
|
||||
{"the retake of a key failed on a transport fault", func(m jetstream.Msg) bool { return retake.heard(ctx, m, logger.Printf) }}, // §568
|
||||
{"what a node's engine reads", func(m jetstream.Msg) bool { return reads.heard(m, logger.Printf) }}, // manifest-reads-changed (ADR 0618 §4)
|
||||
{"the toolchains reported", func(m jetstream.Msg) bool { return heardTools.Reported(ctx, m) }},
|
||||
{"the toolchains running", func(m jetstream.Msg) bool { return heardTools.Running(ctx, m) }},
|
||||
{"the floor", func(m jetstream.Msg) bool { return floor.heard(ctx, m) }},
|
||||
{"the reservations", func(m jetstream.Msg) bool { return reserved.heard(ctx, m) }},
|
||||
{"the registry seats' outcomes", func(m jetstream.Msg) bool { return seatHeard.heard(ctx, m) }},
|
||||
}
|
||||
gone := eventTaker{"the artifacts gone", func(m jetstream.Msg) bool { heardGone(ctx, b, m, logger.Printf); return true }}
|
||||
events := func(m jetstream.Msg) { takeEvent(m, takers, gone, nil, logger.Printf) }
|
||||
// The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, the indexer's take-in.taken and
|
||||
// manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the
|
||||
// catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has
|
||||
@@ -511,10 +512,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
// A change of the seat list judges every plan no seat builds again and says them in one line (design 47 H2 §795).
|
||||
seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx); b.SayUnbuildable(ctx) }, logger.Printf)
|
||||
// Every message the builder's own handlers take is noted on its consumer's ledger, in the state consumers that
|
||||
// builder_health reads (design 47 H2 §794, H-A-435).
|
||||
aged := agedHub{hub: hub, ledgers: &ledgers{logf: logger.Printf, sink: consumersSink(process.AwaitBucket(ctx, js, conn,
|
||||
builds.StateBucket(cred.Module, process.ConsumersState), "builder_health reads no consumer's age", logger.Printf))}}
|
||||
seatGate.Follow(seatlist.ProcessBus(calls.Call, aged), func() { seatsChanged(ctx); b.SayUnbuildable(ctx) }, logger.Printf)
|
||||
half.holders = func() { seatsChanged(ctx) }
|
||||
for _, pattern := range seatEventPatterns {
|
||||
if err := hub.Handle(pattern, events); err != nil {
|
||||
if err := aged.Handle(pattern, events); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -531,4 +531,55 @@ func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) {
|
||||
if store.reads != 2 || !strings.Contains(strings.Join(logs, "\n"), "m05 holds 1 built digests") || !strings.Contains(strings.Join(logs, "\n"), "14 reservations stand at it") {
|
||||
t.Errorf("%d reads for two publisheds, journal %q: want one read each and the line made from it", store.reads, logs)
|
||||
}
|
||||
// Design 47 H2 §794: an event whose handling passed 100 ms is said with how long and the part that took it; one within it
|
||||
// is not. And H-A-435: every message the builder's handlers take is noted on its consumer's ledger, so the state consumers
|
||||
// that builder_health reads holds a message left unacknowledged until it is acknowledged.
|
||||
t.Run("a slow handling is said and a message left unacknowledged is kept", func(t *testing.T) {
|
||||
var said []string
|
||||
clock := time.Unix(1000, 0)
|
||||
now := func() time.Time { return clock }
|
||||
slow := []eventTaker{{"the reservations", func(jetstream.Msg) bool { clock = clock.Add(150 * time.Millisecond); return true }}}
|
||||
fast := []eventTaker{{"the floor", func(jetstream.Msg) bool { clock = clock.Add(100 * time.Millisecond); return true }}}
|
||||
rest := eventTaker{"the artifacts gone", func(jetstream.Msg) bool { return true }}
|
||||
takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
if len(said) != 1 || said[0] != "the event a.published took 150ms to handle, over 100ms: the reservations took it" {
|
||||
t.Errorf("said %q, want one line for the handling over 100 ms", said)
|
||||
}
|
||||
facts := map[string]string{}
|
||||
l := &ledgers{now: now, sink: func(_ context.Context, k string, v []byte) error { facts[k] = string(v); return nil }}
|
||||
hub := &oneHandler{}
|
||||
aged := agedHub{hub: hub, ledgers: l}
|
||||
_ = aged.Handle("x.>", func(m jetstream.Msg) {
|
||||
if m.Subject() == "x.ack" {
|
||||
_ = m.Ack()
|
||||
}
|
||||
})
|
||||
hub.handle(&agedTestMsg{subject: "x.left", seq: 7})
|
||||
if !strings.Contains(facts["builder-events"], `"oldest-unacknowledged":"1970-01-01T00:16:40.25Z"`) {
|
||||
t.Errorf("facts %q, want the message left unacknowledged kept with its delivery", facts)
|
||||
}
|
||||
hub.handle(&agedTestMsg{subject: "x.ack", seq: 8})
|
||||
if !strings.Contains(facts["builder-events"], `"acknowledged":8`) {
|
||||
t.Errorf("facts %q, want the acknowledged message applied", facts)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// oneHandler is an event hub with one handler.
|
||||
type oneHandler struct{ handle func(jetstream.Msg) }
|
||||
|
||||
func (h *oneHandler) Handle(_ string, f func(jetstream.Msg)) error { h.handle = f; return nil }
|
||||
|
||||
// agedTestMsg is a message of the builder's events consumer, with the metadata a ledger reads.
|
||||
type agedTestMsg struct {
|
||||
jetstream.Msg
|
||||
subject string
|
||||
seq uint64
|
||||
}
|
||||
|
||||
func (m *agedTestMsg) Subject() string { return m.subject }
|
||||
func (m *agedTestMsg) Ack() error { return nil }
|
||||
func (m *agedTestMsg) Metadata() (*jetstream.MsgMetadata, error) {
|
||||
return &jetstream.MsgMetadata{Sequence: jetstream.SequencePair{Stream: m.seq}, NumDelivered: 1, Consumer: "builder-events", Stream: "EVENTS"}, nil
|
||||
}
|
||||
|
||||
@@ -14,6 +14,13 @@ import (
|
||||
type countingReserve struct {
|
||||
*reserveMem
|
||||
lists, reads atomic.Int64
|
||||
// indexReads and writes count the reads of an index key and the writes of any key (design 47 H2 §794).
|
||||
indexReads, writes atomic.Int64
|
||||
}
|
||||
|
||||
func (c *countingReserve) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
|
||||
c.writes.Add(1)
|
||||
return c.reserveMem.Update(ctx, key, value, rev)
|
||||
}
|
||||
|
||||
func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string, error) {
|
||||
@@ -24,6 +31,8 @@ func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string,
|
||||
func (c *countingReserve) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
if !strings.HasPrefix(key, "index.") {
|
||||
c.reads.Add(1)
|
||||
} else {
|
||||
c.indexReads.Add(1)
|
||||
}
|
||||
return c.reserveMem.Get(ctx, key)
|
||||
}
|
||||
@@ -52,6 +61,31 @@ func TestTheIndexFindsAPublishedReservationInOneRead(t *testing.T) {
|
||||
store.reads.Store(0)
|
||||
return r, store
|
||||
}
|
||||
// Design 47 H2 §794: 1,000 published events over 2,000 reservations cost one read of the index and one write each.
|
||||
t.Run("1,000 publisheds over 2,000 reservations read the index once and write once each", func(t *testing.T) {
|
||||
store := &countingReserve{reserveMem: &reserveMem{mem: newMem()}}
|
||||
r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
|
||||
for i := range 2000 {
|
||||
m := fmt.Sprintf("r%04d", i)
|
||||
if _, err := r.Reserve(ctx, m, "0.1", []string{m + ":a"}, Source{Fingerprint: "fp" + m}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.Built(ctx, m, "0.1.0", m+":a", "sha256:"+m, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store.indexReads.Store(0)
|
||||
store.writes.Store(0)
|
||||
for i := range 1000 {
|
||||
m := fmt.Sprintf("r%04d", 2*i)
|
||||
if marked, _, err := r.Published(ctx, m+"-a", "0.1.0", "sha256:"+m); err != nil || len(marked) != 1 {
|
||||
t.Fatalf("the published of %s marked %v (%v)", m, marked, err)
|
||||
}
|
||||
}
|
||||
if ir, w, l := store.indexReads.Load(), store.writes.Load(), store.lists.Load(); ir != 1000 || w != 1000 || l != 0 {
|
||||
t.Errorf("1,000 publisheds read the index %d times, wrote %d and listed %d, want 1,000 reads, 1,000 writes and no listing", ir, w, l)
|
||||
}
|
||||
})
|
||||
t.Run("a published by digest reads its one reservation", func(t *testing.T) {
|
||||
r, store := fill(t)
|
||||
marked, _, err := r.Published(ctx, "m10-a", "v0.1.0", "sha256:m10")
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}, {Name: "unbuildable", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
@@ -177,7 +177,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
}
|
||||
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
|
||||
if want := []string{"builder_announce_again", "builder_ask_again", "builder_ask_drop", "builder_asks", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
|
||||
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_unbuildable", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
"builder_floor_again", "builder_floor_waits", "builder_health", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_unbuildable", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes.
|
||||
@@ -244,6 +244,10 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-builder"}) {
|
||||
t.Errorf("process = %v", p)
|
||||
}
|
||||
// Design 47 H2 §794, H-A-435: the health asks builder_health, which names a consumer behind by five minutes.
|
||||
if h, _ := p["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "builder_health" {
|
||||
t.Errorf("health = %v, want the tool builder_health", p["health"])
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
|
||||
|
||||
@@ -38,6 +38,7 @@
|
||||
"builder_toolchains_drop",
|
||||
"builder_waits",
|
||||
"builder_unbuildable",
|
||||
"builder_health",
|
||||
"builder_ask_drop",
|
||||
"builder_asks",
|
||||
"builder_versions",
|
||||
@@ -142,7 +143,8 @@
|
||||
{ "name": "reasks", "history": 1 },
|
||||
{ "name": "versions", "history": 1 },
|
||||
{ "name": "first-views", "history": 1 },
|
||||
{ "name": "unbuildable", "history": 1, "max-mib": 8 }
|
||||
{ "name": "unbuildable", "history": 1, "max-mib": 8 },
|
||||
{ "name": "consumers", "history": 1, "max-mib": 8 }
|
||||
],
|
||||
"streams": [
|
||||
{ "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 },
|
||||
@@ -344,7 +346,8 @@
|
||||
},
|
||||
"user": "mesh-delivery-builder",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
"kind": "tool",
|
||||
"tool": "builder_health"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user