Design 47 H2: the delivery parts' classes and caps, the builder's publish queue, the indexer's seat list, the runner and the forge bridge #322

Merged
mesh-tools merged 121 commits from feat/47-H2 into main 2026-11-22 04:56:13 +00:00
7 changed files with 101 additions and 25 deletions
Showing only changes of commit d68e71700a - Show all commits
@@ -4,6 +4,7 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-builder/internal/versions"
)
@@ -11,20 +12,29 @@ import (
const (
// HealthTool is builder_health, the builder's health as its manifest's health asks it (design 47 H2 §794, H-A-435).
HealthTool = "builder_health"
// BehindAfter is how long a durable consumer's oldest unacknowledged message may wait before the health names it: an
// events backlog, or a handler standing still on one message (§794).
BehindAfter = 5 * time.Minute
// eventsBehindBoundSetting is the setting events-behind-bound (§794; H-A-461 (2), P11): how long a durable consumer's
// oldest unacknowledged message may wait before the health names it, an events backlog or a handler standing still on one
// message; defaultEventsBehindBound when none is set.
eventsBehindBoundSetting = "events-behind-bound"
defaultEventsBehindBound = 5 * time.Minute
)
// healthTool is builder_health: healthy when it answers, and each of the builder's durable consumers' oldest unacknowledged
// message waits less than BehindAfter, as the consumers' facts the builder writes at every delivery and every acknowledgement
// say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise unhealthy, naming each
// consumer behind and how long.
// message waits less than the setting events-behind-bound, as the consumers' facts the builder writes at every delivery and
// every acknowledgement say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise
// unhealthy, naming each consumer behind and how long. A setting that holds no duration is said as the health's answer.
func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool {
return stdio.Tool{Name: HealthTool,
Description: "The builder's health: healthy when it answers and each of its durable consumers' oldest unacknowledged message waits less " +
"than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " +
"Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.",
"than the setting events-behind-bound (five minutes by default; no events backlog, no handler standing still on a message); otherwise it " +
"names each consumer behind and how long. Read from the builder's state consumers, which it writes at every message delivered and " +
"acknowledged. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return versions.BuilderHealth(consumers, now(), BehindAfter), nil }}
Run: func(map[string]any) (any, error) {
bound, err := stdio.SettingDuration(eventsBehindBoundSetting, defaultEventsBehindBound)
if err != nil {
return health.Unhealthy("The builder's " + err.Error() + "."), nil
}
return versions.BuilderHealth(consumers, now(), bound), nil
}}
}
@@ -11,6 +11,7 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-builder/internal/build"
@@ -58,13 +59,23 @@ func TestAStateIsReadInOnePage(t *testing.T) {
"builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)}
r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil },
Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }}
a := versions.BuilderHealth(r, now, BehindAfter)
a := versions.BuilderHealth(r, now, defaultEventsBehindBound)
if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." {
t.Errorf("health = %+v, want unhealthy naming builder-events alone", a)
}
if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), BehindAfter); !a.Healthy {
if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), defaultEventsBehindBound); !a.Healthy {
t.Errorf("health four minutes in = %+v, want healthy", a)
}
// H-A-461 (2): the bound is the setting events-behind-bound, five minutes when none is set.
tool := healthTool(r, func() time.Time { return now })
for _, c := range []struct{ set, want string }{{"", "unhealthy"}, {"10m", "healthy"}, {"soon", "events-behind-bound"}} {
t.Setenv(stdio.SettingWord(eventsBehindBoundSetting), c.set)
got, _ := tool.Run(nil)
a := got.(health.Answer)
if (c.want == "healthy") != a.Healthy || (c.want == "events-behind-bound" && !strings.Contains(a.Why, c.want)) {
t.Errorf("with events-behind-bound %q the health = %+v, want %s", c.set, a, c.want)
}
}
})
// Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and
@@ -1,14 +1,30 @@
package main
import (
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// SlowEvent is how long the handling of one heard event may take before the builder says it in its journal (design 47 H2
// §794): the line names the event's subject, the part that took it and how long it took.
const SlowEvent = 100 * time.Millisecond
// slowEventSetting is the setting slow-event (design 47 H2 §794; H-A-461 (2), P11): how long the handling of one heard event
// may take before the builder says it in its journal, the line naming the event's subject, the part that took it and how long
// it took; defaultSlowEvent when none is set.
const (
slowEventSetting = "slow-event"
defaultSlowEvent = 100 * time.Millisecond
)
// slowEvent reads the setting slow-event.
func slowEvent() (time.Duration, error) {
d, err := stdio.SettingDuration(slowEventSetting, defaultSlowEvent)
if err != nil {
return 0, errors.New("the builder's " + err.Error())
}
return d, nil
}
// eventTaker is one part of the builder that takes events off its durable consumer: take answers whether the message was its.
type eventTaker struct {
@@ -16,9 +32,9 @@ type eventTaker struct {
take func(jetstream.Msg) bool
}
// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed SlowEvent
// (design 47 H2 §794). now is the clock, a test's when set.
func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() time.Time, logf func(string, ...any)) {
// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed slow, the
// setting slow-event (design 47 H2 §794). now is the clock, a test's when set.
func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, slow time.Duration, now func() time.Time, logf func(string, ...any)) {
if now == nil {
now = time.Now
}
@@ -33,7 +49,7 @@ func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func()
if part == rest.part {
rest.take(m)
}
if took := now().Sub(start); took > SlowEvent {
logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), SlowEvent, part)
if took := now().Sub(start); took > slow {
logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), slow, part)
}
}
@@ -479,7 +479,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf}
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
// Each part takes the events it hears in turn; a handling over SlowEvent is said with the part that took it (design 47 H2 §794).
// Each part takes the events it hears in turn; a handling over the setting slow-event is said with the part that took it
// (design 47 H2 §794; H-A-461 (2)).
slow, err := slowEvent()
if err != nil {
return err
}
takers := []eventTaker{
{"the operator's answer to an ask about a key", decided.heard}, // §568
{"the retake of a key failed on a transport fault", func(m jetstream.Msg) bool { return retake.heard(ctx, m, logger.Printf) }}, // §568
@@ -491,7 +496,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
{"the registry seats' outcomes", func(m jetstream.Msg) bool { return seatHeard.heard(ctx, m) }},
}
gone := eventTaker{"the artifacts gone", func(m jetstream.Msg) bool { heardGone(ctx, b, m, logger.Printf); return true }}
events := func(m jetstream.Msg) { takeEvent(m, takers, gone, nil, logger.Printf) }
events := func(m jetstream.Msg) { takeEvent(m, takers, gone, slow, nil, logger.Printf) }
// The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, the indexer's take-in.taken and
// manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the
// catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has
@@ -16,6 +16,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
@@ -587,11 +588,27 @@ func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) {
slow := []eventTaker{{"the reservations", func(jetstream.Msg) bool { clock = clock.Add(150 * time.Millisecond); return true }}}
fast := []eventTaker{{"the floor", func(jetstream.Msg) bool { clock = clock.Add(100 * time.Millisecond); return true }}}
rest := eventTaker{"the artifacts gone", func(jetstream.Msg) bool { return true }}
takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
if len(said) != 1 || said[0] != "the event a.published took 150ms to handle, over 100ms: the reservations took it" {
t.Errorf("said %q, want one line for the handling over 100 ms", said)
}
// H-A-461 (2): the bound is the setting slow-event, 100 ms when none is set; at 200ms the 150 ms handling is not said.
for _, c := range []struct {
set string
want time.Duration
}{{"", defaultSlowEvent}, {"200ms", 200 * time.Millisecond}, {"soon", 0}} {
t.Setenv(stdio.SettingWord(slowEventSetting), c.set)
if d, err := slowEvent(); d != c.want || (c.want == 0) != (err != nil) {
t.Errorf("with slow-event %q the bound is %v (%v), want %v", c.set, d, err, c.want)
}
}
said = nil
takeEvent(&agedTestMsg{subject: "a.published", seq: 3}, slow, rest, 200*time.Millisecond, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
if len(said) != 0 {
t.Errorf("said %q with slow-event 200ms, want nothing for a 150 ms handling", said)
}
clock = clock.Add(-150 * time.Millisecond) // the ledger's times below read as before this row
facts := map[string]string{}
l := &ledgers{now: now, sink: func(_ context.Context, k string, v []byte) error { facts[k] = string(v); return nil }}
hub := &oneHandler{}
@@ -200,10 +200,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 {
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
}
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 10 ||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 ||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime ||
// H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default
m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime ||
m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 {
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
}
+14
View File
@@ -204,6 +204,20 @@
"default": "30s",
"runtime": true
},
"slow-event": {
"type": "string",
"description": "how long the handling of one heard event may take before the builder says it in its journal, with its duration and the part that took it, a Go duration (design 47 H2 §794; H-A-461 (2)).",
"trusted": false,
"default": "100ms",
"runtime": true
},
"events-behind-bound": {
"type": "string",
"description": "how long a durable consumer's oldest unacknowledged message may wait before builder_health names it, an events backlog or a handler standing still, a Go duration (design 47 H2 §794; H-A-461 (2)).",
"trusted": false,
"default": "5m",
"runtime": true
},
"publish-pause": {
"type": "number",
"description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).",