R7: the registration check waits for its view; runner_health answers registrations-read
Builds: design 47 H2 §788
This commit is contained in:
@@ -13,6 +13,7 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
@@ -102,16 +103,22 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now
|
||||
// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A
|
||||
// running pipeline it first names stalled is said in this program's journal through logf, once per key for the
|
||||
// program's life (H-A-311 (3), H-A-313).
|
||||
func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool {
|
||||
func HealthTool(pipelines, consumers, firstViews pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool {
|
||||
var mu sync.Mutex
|
||||
said := map[string]bool{}
|
||||
return stdio.Tool{Name: Prefix + "_health",
|
||||
Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " +
|
||||
"unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " +
|
||||
"newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key.",
|
||||
"newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key. It " +
|
||||
"answers registrations-read: whether the runner's registrations view's first read is in, and since when.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
a, stalled := pipeline.HealthStalled(pipelines, consumers, now())
|
||||
// design 47 H2 §788: whether the registrations are read, from the runner's own first-views state.
|
||||
read, err := pipeline.RegistrationsReadOf(firstViews)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, key := range stalled {
|
||||
@@ -120,10 +127,16 @@ func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf f
|
||||
logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter)
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
return HealthAnswer{Answer: a, RegistrationsRead: read}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// HealthAnswer is runner_health's answer: the health and registrations-read (design 47 H2 §788).
|
||||
type HealthAnswer struct {
|
||||
health.Answer
|
||||
RegistrationsRead pipeline.RegistrationsRead `json:"registrations-read"`
|
||||
}
|
||||
|
||||
// StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states
|
||||
// pipelines and kept by `mesh/state.info`, step-outputs by the object store's fill and step-log by the log's; the runner's
|
||||
// process writes nothing for it. A store whose fill cannot be read is an error naming it.
|
||||
@@ -172,7 +185,7 @@ func main() {
|
||||
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
|
||||
tools = append(tools, VerbTools(stdio.Ask)...)
|
||||
tools = append(tools, TestRunsTools(stdio.Ask)...)
|
||||
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now,
|
||||
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, kept{stdio.State(views.StatusState)}, time.Now,
|
||||
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) }))
|
||||
tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)}))
|
||||
tools = append(tools, UnfinishedTools(keptDelete{kept{stdio.State(pipeline.StateUnfinished)}}, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
|
||||
@@ -42,7 +42,7 @@ func TestTools(t *testing.T) {
|
||||
names = append(names, tl.Name)
|
||||
}
|
||||
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
|
||||
names = append(names, HealthTool(none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name)
|
||||
names = append(names, HealthTool(none{}, none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name)
|
||||
// §606, §549: the unlogged entries are listed and appended by hand.
|
||||
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
|
||||
names = append(names, tl.Name)
|
||||
@@ -70,7 +70,8 @@ func TestTools(t *testing.T) {
|
||||
quiet := queueOf{"main.137": `{"pipeline":"main.137","branch":"main","outcome":"running","source":137,"started":"2026-11-21T20:15:00Z",` +
|
||||
`"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`}
|
||||
var journal []string
|
||||
health := HealthTool(quiet, none{}, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
|
||||
firstViews := queueOf{}
|
||||
health := HealthTool(quiet, none{}, firstViews, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
|
||||
for range 2 {
|
||||
out, err := health.Run(nil)
|
||||
if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") {
|
||||
@@ -80,6 +81,17 @@ func TestTools(t *testing.T) {
|
||||
if len(journal) != 1 || !strings.Contains(journal[0], "main.137") {
|
||||
t.Errorf("the journal said %q, want main.137 said once", journal)
|
||||
}
|
||||
// design 47 H2 §788: registrations-read is false, since the ask, until the registrations' first view is answered.
|
||||
for _, c := range []struct{ status, want string }{
|
||||
{`{"owner":"mesh-catalog","state":"registrations","asked":"2026-11-21T20:30:00Z"}`, `"registrations-read":{"read":false,"since":"2026-11-21T20:30:00Z"}`},
|
||||
{`{"owner":"mesh-catalog","state":"registrations","asked":"2026-11-21T20:30:00Z","answered":"2026-11-21T20:30:02Z"}`, `"registrations-read":{"read":true,"since":"2026-11-21T20:30:02Z"}`},
|
||||
} {
|
||||
firstViews["mesh-catalog.registrations"] = c.status
|
||||
out, err := health.Run(nil)
|
||||
if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), c.want) {
|
||||
t.Errorf("runner_health = %s (%v), want %s", got, err, c.want)
|
||||
}
|
||||
}
|
||||
// design 47 H2 §786: runner_unfinished reads each key and its entry; runner_unfinished_drop names why, drops nothing when
|
||||
// its hand act is not recorded, and drops the key once it is.
|
||||
if got, err := tools[0].Run(nil); err != nil || fmt.Sprint(got) != "[{main.app {src2:aa 4 main.4 failed}}]" {
|
||||
|
||||
@@ -169,7 +169,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
|
||||
// for its step.
|
||||
starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say,
|
||||
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, publish: js.PublishMsg, cred: cred, bound: askBound, logf: logger.Printf}, Name: cred.Module,
|
||||
Registration: rv.Registrations.Registration, RegistrationsRead: func() bool { return chanClosed(rv.Registrations.Ready()) }, Steps: busStepAsker{ctx: ctx, publish: js.PublishMsg, cred: cred, bound: askBound, logf: logger.Printf}, Name: cred.Module,
|
||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing,
|
||||
Findings: rv.Findings, Observer: cred.Module, // a mover waiting on its registration is registration-missing (H-A-262)
|
||||
@@ -397,3 +397,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
|
||||
// ownBucket is a state of the runner's old state section on the bus, `<module>_<state>`.
|
||||
func ownBucket(module, state string) string { return module + "_" + state }
|
||||
|
||||
// chanClosed says whether c is closed now, without waiting.
|
||||
func chanClosed(c <-chan struct{}) bool {
|
||||
select {
|
||||
case <-c:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,9 +7,11 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// StateConsumers is the runner's old state section of its durable consumers' facts, a key per consumer, written by
|
||||
@@ -128,3 +130,35 @@ func consumersHealth(consumers Kept, now time.Time) health.Answer {
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("The runner answers show, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter))
|
||||
}
|
||||
|
||||
// RegistrationsRead is runner_health's registrations-read (design 47 H2 §788): whether the runner's registrations view's
|
||||
// first read is in, and since when; before it, since when it was asked.
|
||||
type RegistrationsRead struct {
|
||||
Read bool `json:"read"`
|
||||
Since *time.Time `json:"since,omitempty"`
|
||||
}
|
||||
|
||||
// RegistrationsReadOf reads registrations-read from the runner's own first-views state, where its process keeps each first
|
||||
// view's status (views.StatusState): read once the seat mesh-catalog's registrations view was answered.
|
||||
func RegistrationsReadOf(firstViews Kept) (RegistrationsRead, error) {
|
||||
raw, err := firstViews.Value(views.StatusKey(catalog.Seat, catalog.StateRegistrations))
|
||||
if err != nil {
|
||||
return RegistrationsRead{}, fmt.Errorf("the first view of the registrations is not read from %s: %w", views.StatusState, err)
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return RegistrationsRead{}, nil
|
||||
}
|
||||
var st views.Status
|
||||
if err := json.Unmarshal(raw, &st); err != nil {
|
||||
return RegistrationsRead{}, fmt.Errorf("the first view of the registrations in %s: %w", views.StatusState, err)
|
||||
}
|
||||
if !st.Answered.IsZero() {
|
||||
at := st.Answered
|
||||
return RegistrationsRead{Read: true, Since: &at}, nil
|
||||
}
|
||||
if !st.Asked.IsZero() {
|
||||
at := st.Asked
|
||||
return RegistrationsRead{Since: &at}, nil
|
||||
}
|
||||
return RegistrationsRead{}, nil
|
||||
}
|
||||
|
||||
@@ -20,6 +20,13 @@ func registrationMissingID(module string) string { return keys.EscapeName(module
|
||||
|
||||
// registrationMissing raises registration-missing of module, waited on by the pipeline key (H-A-262).
|
||||
func (s *Starter) registrationMissing(module, key string) {
|
||||
// Judged only once the registrations view's first read is in (design 47 H2 §788); before it, said once.
|
||||
if s.RegistrationsRead != nil && !s.RegistrationsRead() {
|
||||
if s.unreadSaid.CompareAndSwap(false, true) {
|
||||
s.logf("registrations not yet read: no step is placed from a registration not read, and no %s is judged before they are (%s waits, in %s)", RegistrationMissingKind, module, key)
|
||||
}
|
||||
return
|
||||
}
|
||||
if s.Findings == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
@@ -162,6 +163,10 @@ type Starter struct {
|
||||
// Registration is a module's registration, whose policy names the steps of its pipeline (§21, §23); with none, a
|
||||
// pipeline starts with no step and says so.
|
||||
Registration func(module string) (catalog.Registration, bool)
|
||||
// RegistrationsRead says whether the registrations view's first read is in (design 47 H2 §788): before it no
|
||||
// registration-missing is judged and "registrations not yet read" is said once; nil reads as read.
|
||||
RegistrationsRead func() bool
|
||||
unreadSaid atomic.Bool
|
||||
// Steps asks each build seat for its step, as the pipeline's asking half (§22, §24); Name is the runner's own name as an
|
||||
// asker.
|
||||
Steps StepAsker
|
||||
|
||||
@@ -197,6 +197,45 @@ func TestATakeInIsOnePipeline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// design 47 H2 §788: before the registrations view's first read is in, a start raises no registration-missing, says
|
||||
// "registrations not yet read" once and places no step; once read, its movers are placed from the registrations held.
|
||||
func TestTheRegistrationCheckWaitsForItsView(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
store := memCreated{memStore{}}
|
||||
asks := &stepAsks{}
|
||||
read := false
|
||||
var lines []string
|
||||
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil },
|
||||
Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) },
|
||||
RegistrationsRead: func() bool { return read },
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, read && m == "app" }}
|
||||
findings := newHeld()
|
||||
s.Findings, s.Observer = findings, "the-runner"
|
||||
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
|
||||
for _, src := range []int64{4, 5} {
|
||||
taken := catalog.TakeInTaken{Branch: "main", Source: src, Entries: []catalog.TakeInEntry{{Module: "app", Commit: fmt.Sprint("c", src), Fingerprint: fmt.Sprint("src2:a", src)}}}
|
||||
if _, err := s.TakenIn(taken, view); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
said := 0
|
||||
for _, l := range lines {
|
||||
if strings.HasPrefix(l, "registrations not yet read") {
|
||||
said++
|
||||
}
|
||||
}
|
||||
if len(findings.held) != 0 || said != 1 || len(asks.asks) != 0 {
|
||||
t.Fatalf("unread: findings %v, said %d times, %d asks; want none, once, none", findings.held, said, len(asks.asks))
|
||||
}
|
||||
read = true
|
||||
if n, err := s.Waiting(store.memStore, "", view); n == 0 || err != nil || len(asks.asks) == 0 {
|
||||
t.Fatalf("once read: placed in %d (%v), %d asks", n, err, len(asks.asks))
|
||||
}
|
||||
if len(findings.held) != 0 {
|
||||
t.Fatalf("once read, a registered mover raised %v", findings.held)
|
||||
}
|
||||
}
|
||||
|
||||
// A-327 (design 47 G §21, §23, §54): a pipeline started before its module has a registration starts with no step; the
|
||||
// registration-changed that brings it places that pipeline's steps from it and asks them, once: heard again it places nothing.
|
||||
func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user