R7: the registration check waits for its view; runner_health answers registrations-read

Builds: design 47 H2 §788
This commit is contained in:
2026-11-22 01:19:27 +01:00
parent 8d43ce0cb9
commit 43a88ef96c
7 changed files with 127 additions and 7 deletions
@@ -13,6 +13,7 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/busstate"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/views"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
@@ -102,16 +103,22 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now
// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A
// running pipeline it first names stalled is said in this program's journal through logf, once per key for the
// program's life (H-A-311 (3), H-A-313).
func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool {
func HealthTool(pipelines, consumers, firstViews pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool {
var mu sync.Mutex
said := map[string]bool{}
return stdio.Tool{Name: Prefix + "_health",
Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " +
"unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " +
"newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key.",
"newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key. It " +
"answers registrations-read: whether the runner's registrations view's first read is in, and since when.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
a, stalled := pipeline.HealthStalled(pipelines, consumers, now())
// design 47 H2 §788: whether the registrations are read, from the runner's own first-views state.
read, err := pipeline.RegistrationsReadOf(firstViews)
if err != nil {
return nil, err
}
mu.Lock()
defer mu.Unlock()
for _, key := range stalled {
@@ -120,10 +127,16 @@ func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf f
logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter)
}
}
return a, nil
return HealthAnswer{Answer: a, RegistrationsRead: read}, nil
}}
}
// HealthAnswer is runner_health's answer: the health and registrations-read (design 47 H2 §788).
type HealthAnswer struct {
health.Answer
RegistrationsRead pipeline.RegistrationsRead `json:"registrations-read"`
}
// StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states
// pipelines and kept by `mesh/state.info`, step-outputs by the object store's fill and step-log by the log's; the runner's
// process writes nothing for it. A store whose fill cannot be read is an error naming it.
@@ -172,7 +185,7 @@ func main() {
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
tools = append(tools, VerbTools(stdio.Ask)...)
tools = append(tools, TestRunsTools(stdio.Ask)...)
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now,
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, kept{stdio.State(views.StatusState)}, time.Now,
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) }))
tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)}))
tools = append(tools, UnfinishedTools(keptDelete{kept{stdio.State(pipeline.StateUnfinished)}}, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
@@ -42,7 +42,7 @@ func TestTools(t *testing.T) {
names = append(names, tl.Name)
}
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
names = append(names, HealthTool(none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name)
names = append(names, HealthTool(none{}, none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name)
// §606, §549: the unlogged entries are listed and appended by hand.
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
names = append(names, tl.Name)
@@ -70,7 +70,8 @@ func TestTools(t *testing.T) {
quiet := queueOf{"main.137": `{"pipeline":"main.137","branch":"main","outcome":"running","source":137,"started":"2026-11-21T20:15:00Z",` +
`"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`}
var journal []string
health := HealthTool(quiet, none{}, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
firstViews := queueOf{}
health := HealthTool(quiet, none{}, firstViews, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
for range 2 {
out, err := health.Run(nil)
if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") {
@@ -80,6 +81,17 @@ func TestTools(t *testing.T) {
if len(journal) != 1 || !strings.Contains(journal[0], "main.137") {
t.Errorf("the journal said %q, want main.137 said once", journal)
}
// design 47 H2 §788: registrations-read is false, since the ask, until the registrations' first view is answered.
for _, c := range []struct{ status, want string }{
{`{"owner":"mesh-catalog","state":"registrations","asked":"2026-11-21T20:30:00Z"}`, `"registrations-read":{"read":false,"since":"2026-11-21T20:30:00Z"}`},
{`{"owner":"mesh-catalog","state":"registrations","asked":"2026-11-21T20:30:00Z","answered":"2026-11-21T20:30:02Z"}`, `"registrations-read":{"read":true,"since":"2026-11-21T20:30:02Z"}`},
} {
firstViews["mesh-catalog.registrations"] = c.status
out, err := health.Run(nil)
if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), c.want) {
t.Errorf("runner_health = %s (%v), want %s", got, err, c.want)
}
}
// design 47 H2 §786: runner_unfinished reads each key and its entry; runner_unfinished_drop names why, drops nothing when
// its hand act is not recorded, and drops the key once it is.
if got, err := tools[0].Run(nil); err != nil || fmt.Sprint(got) != "[{main.app {src2:aa 4 main.4 failed}}]" {
@@ -169,7 +169,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
// for its step.
starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say,
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, publish: js.PublishMsg, cred: cred, bound: askBound, logf: logger.Printf}, Name: cred.Module,
Registration: rv.Registrations.Registration, RegistrationsRead: func() bool { return chanClosed(rv.Registrations.Ready()) }, Steps: busStepAsker{ctx: ctx, publish: js.PublishMsg, cred: cred, bound: askBound, logf: logger.Printf}, Name: cred.Module,
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing,
Findings: rv.Findings, Observer: cred.Module, // a mover waiting on its registration is registration-missing (H-A-262)
@@ -397,3 +397,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// ownBucket is a state of the runner's old state section on the bus, `<module>_<state>`.
func ownBucket(module, state string) string { return module + "_" + state }
// chanClosed says whether c is closed now, without waiting.
func chanClosed(c <-chan struct{}) bool {
select {
case <-c:
return true
default:
return false
}
}
@@ -7,9 +7,11 @@ import (
"strings"
"time"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
)
// StateConsumers is the runner's old state section of its durable consumers' facts, a key per consumer, written by
@@ -128,3 +130,35 @@ func consumersHealth(consumers Kept, now time.Time) health.Answer {
}
return health.Healthy(fmt.Sprintf("The runner answers show, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter))
}
// RegistrationsRead is runner_health's registrations-read (design 47 H2 §788): whether the runner's registrations view's
// first read is in, and since when; before it, since when it was asked.
type RegistrationsRead struct {
Read bool `json:"read"`
Since *time.Time `json:"since,omitempty"`
}
// RegistrationsReadOf reads registrations-read from the runner's own first-views state, where its process keeps each first
// view's status (views.StatusState): read once the seat mesh-catalog's registrations view was answered.
func RegistrationsReadOf(firstViews Kept) (RegistrationsRead, error) {
raw, err := firstViews.Value(views.StatusKey(catalog.Seat, catalog.StateRegistrations))
if err != nil {
return RegistrationsRead{}, fmt.Errorf("the first view of the registrations is not read from %s: %w", views.StatusState, err)
}
if len(raw) == 0 {
return RegistrationsRead{}, nil
}
var st views.Status
if err := json.Unmarshal(raw, &st); err != nil {
return RegistrationsRead{}, fmt.Errorf("the first view of the registrations in %s: %w", views.StatusState, err)
}
if !st.Answered.IsZero() {
at := st.Answered
return RegistrationsRead{Read: true, Since: &at}, nil
}
if !st.Asked.IsZero() {
at := st.Asked
return RegistrationsRead{Since: &at}, nil
}
return RegistrationsRead{}, nil
}
@@ -20,6 +20,13 @@ func registrationMissingID(module string) string { return keys.EscapeName(module
// registrationMissing raises registration-missing of module, waited on by the pipeline key (H-A-262).
func (s *Starter) registrationMissing(module, key string) {
// Judged only once the registrations view's first read is in (design 47 H2 §788); before it, said once.
if s.RegistrationsRead != nil && !s.RegistrationsRead() {
if s.unreadSaid.CompareAndSwap(false, true) {
s.logf("registrations not yet read: no step is placed from a registration not read, and no %s is judged before they are (%s waits, in %s)", RegistrationMissingKind, module, key)
}
return
}
if s.Findings == nil {
return
}
@@ -9,6 +9,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
@@ -162,6 +163,10 @@ type Starter struct {
// Registration is a module's registration, whose policy names the steps of its pipeline (§21, §23); with none, a
// pipeline starts with no step and says so.
Registration func(module string) (catalog.Registration, bool)
// RegistrationsRead says whether the registrations view's first read is in (design 47 H2 §788): before it no
// registration-missing is judged and "registrations not yet read" is said once; nil reads as read.
RegistrationsRead func() bool
unreadSaid atomic.Bool
// Steps asks each build seat for its step, as the pipeline's asking half (§22, §24); Name is the runner's own name as an
// asker.
Steps StepAsker
@@ -197,6 +197,45 @@ func TestATakeInIsOnePipeline(t *testing.T) {
}
}
// design 47 H2 §788: before the registrations view's first read is in, a start raises no registration-missing, says
// "registrations not yet read" once and places no step; once read, its movers are placed from the registrations held.
func TestTheRegistrationCheckWaitsForItsView(t *testing.T) {
reg, entries := placeFixture()
store := memCreated{memStore{}}
asks := &stepAsks{}
read := false
var lines []string
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil },
Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) },
RegistrationsRead: func() bool { return read },
Registration: func(m string) (catalog.Registration, bool) { return reg, read && m == "app" }}
findings := newHeld()
s.Findings, s.Observer = findings, "the-runner"
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
for _, src := range []int64{4, 5} {
taken := catalog.TakeInTaken{Branch: "main", Source: src, Entries: []catalog.TakeInEntry{{Module: "app", Commit: fmt.Sprint("c", src), Fingerprint: fmt.Sprint("src2:a", src)}}}
if _, err := s.TakenIn(taken, view); err != nil {
t.Fatal(err)
}
}
said := 0
for _, l := range lines {
if strings.HasPrefix(l, "registrations not yet read") {
said++
}
}
if len(findings.held) != 0 || said != 1 || len(asks.asks) != 0 {
t.Fatalf("unread: findings %v, said %d times, %d asks; want none, once, none", findings.held, said, len(asks.asks))
}
read = true
if n, err := s.Waiting(store.memStore, "", view); n == 0 || err != nil || len(asks.asks) == 0 {
t.Fatalf("once read: placed in %d (%v), %d asks", n, err, len(asks.asks))
}
if len(findings.held) != 0 {
t.Fatalf("once read, a registered mover raised %v", findings.held)
}
}
// A-327 (design 47 G §21, §23, §54): a pipeline started before its module has a registration starts with no step; the
// registration-changed that brings it places that pipeline's steps from it and asks them, once: heard again it places nothing.
func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T) {