From e583dce741fac3f215272acc46c36ac7ad14aee0 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 21:25:03 +0100 Subject: [PATCH] H-A-326 (1), (3): a mover's wait on a package is a read first, an event second main.137 stood twenty minutes with nats and mesh-vault waiting on mesh-grants packages published at 20:00, before the wait was written, so no publish event came. The runner now asks the build seat's published when it writes the wait and places the mover when the package is published, saying so in the journal apart from an event. Builds: design 47 H H-A-326 (1), H-A-326 (3), H-A-328 --- .../cmd/mesh-delivery-pipeline/main.go | 2 + .../cmd/mesh-delivery-pipeline/published.go | 29 ++++++++++ .../internal/pipeline/order_test.go | 34 ++++++++++++ .../internal/pipeline/repin.go | 3 ++ .../internal/pipeline/start.go | 54 +++++++++++++++++++ .../mesh-delivery-pipeline/manifest_test.go | 2 + modules/mesh-delivery-pipeline/module.json | 1 + 7 files changed, 125 insertions(+) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 19e9c98a5..0e6ecb14e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -179,6 +179,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) } starter.Left = hearer.LeftInPlace // a same-source re-pin's superseded steps leave their runs (H-A-284) starter.Revision = widener.Revision // a kept mover is placed from its kept revision (H-A-286) + // A mover's wait on a package is read first: a package published before the wait places it at once (H-A-326, H-A-328). + starter.AskPublished = publishedOf(boundedCalls(calls.Call, readBound, time.After, logger.Printf)) hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } hearer.AskPinHeld = func(key, module string, pins map[string]string) { if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go new file mode 100644 index 000000000..1b4fbdbb2 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "fmt" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// publishedOf asks the seat mesh-builds' verb published through call whether a package built at a fingerprint is published +// (design 47 H-A-326 (1), H-A-328), asked when a mover's wait on it is written. +func publishedOf(call func(string, any) (json.RawMessage, error)) func(builds.PublishedInput) (builds.PublishedAnswer, error) { + return func(in builds.PublishedInput) (builds.PublishedAnswer, error) { + raw, err := call("seat:"+builds.Seat+"."+builds.VerbPublished, in) + if err != nil { + return builds.PublishedAnswer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return builds.PublishedAnswer{}, fmt.Errorf("%s answered: %s", builds.VerbPublished, said) + } + var a builds.PublishedAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return builds.PublishedAnswer{}, fmt.Errorf("%s answered what does not read: %w", builds.VerbPublished, err) + } + return a, nil + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 4d4aeff0c..75edef4d0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "fmt" "slices" "strings" "testing" @@ -141,6 +142,39 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { t.Errorf("tool's step-ended = %+v, want not asked naming builder", ev.Result) } } + + // H-A-326 (1), (3), H-A-328: a package published before its importer's wait was written (by an earlier pipeline) is read of + // the build seat when the wait is written, matched by its mover's fingerprint, and the importer is placed with no event; + // the journal says the wait was resolved from the build seat's published, and a retried placing asks it once. + mem2 := memStore{} + var asked []builds.PublishedInput + var journal []string + s2 := &Starter{Pipelines: memCreated{mem2}, Store: mem2, Say: func(string, any) error { return nil }, Steps: &stepAsks{}, Name: "the-runner", + Registration: func(string) (catalog.Registration, bool) { return reg, true }, + Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }, + AskPublished: func(in builds.PublishedInput) (builds.PublishedAnswer, error) { + asked = append(asked, in) + return builds.PublishedAnswer{Published: in.Artifact == "mesh-sdk:go" && in.Fingerprint == "src2:00", Version: "0.2.0"}, nil + }} + if n, err := s2.TakenIn(ev, view); n != 1 || err != nil { + t.Fatalf("the take-in started %d (%v)", n, err) + } + p, _, _ = mem2.Get("main.7") + if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || !slices.Contains(sdk.Published, "mesh-sdk:go") { + t.Errorf("with mesh-sdk:go published before the wait: app placed %v, tool placed %v, mesh-sdk %+v; want app placed with no event", placed(p, "app"), placed(p, "tool"), sdk) + } + if !slices.ContainsFunc(journal, func(l string) bool { + return strings.Contains(l, "app's wait on mesh-sdk:go") && strings.Contains(l, "placed from the published record") + }) { + t.Errorf("the journal %q does not say app's wait was resolved from the build seat's published", journal) + } + before2 := len(asked) + if _, err := s2.Waiting(mem2, "main", view); err != nil { + t.Fatal(err) + } + if len(asked) != before2 { + t.Errorf("a placing again asked published %d more times, want each package asked once", len(asked)-before2) + } } // Design 47 G §270, A-326 (H §746, H-A-83 (2)): what a build module's new version is built from is built by its running diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/repin.go b/modules/mesh-delivery-pipeline/internal/pipeline/repin.go index a13a850ff..93a2dcc18 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/repin.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/repin.go @@ -177,6 +177,9 @@ func (s *Starter) placeAgain(p *seat.Pipeline, m seat.Mover, half Half, now time return nil, nil, true } due, waiting, failed := ready(*p, m.Module, half.Entries) + if !due && failed == "" && s.readPublished(p, m.Module, waiting) { + due, waiting, failed = ready(*p, m.Module, half.Entries) + } if !due && failed == "" { s.logf("%s: %s waits for %s before its build", p.Pipeline, m.Module, strings.Join(waiting, ", ")) return nil, nil, false diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 4469a8099..62926dc07 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" @@ -199,6 +200,56 @@ type Starter struct { // registration-missing (H-A-262); Observer is the runner's own module name. nil raises none. Findings Findings Observer string + // AskPublished asks the seat mesh-builds' verb published whether a package built at a fingerprint is published (H-A-326 + // (1), H-A-328): read when a mover's wait on it is written, so a publish made before the wait places the mover without + // any event; nil reads nothing, the publish event alone placing it. + AskPublished func(in builds.PublishedInput) (builds.PublishedAnswer, error) + + publishedMu sync.Mutex + publishedRead map[string]builds.PublishedAnswer +} + +// readPublished asks, once per pipeline, package and fingerprint, the seat mesh-builds whether each package mover waits on +// in p is published already (H-A-326 (1), H-A-328): a wait is a read first and an event second. Each package answered +// published goes into its From mover's published in p, written with the placing, and the journal says the wait resolved +// from the build seat's published, apart from one a package-published places (H-A-326 (3)). The answer is kept for the +// placing's compare-and-set retries, so no retry asks again. It says whether it added any. +func (s *Starter) readPublished(p *seat.Pipeline, mover string, waiting []string) bool { + if s.AskPublished == nil { + return false + } + added := false + for _, ref := range waiting { + i := slices.IndexFunc(p.Movers, func(m seat.Mover) bool { return m.Module == moduleOf(ref) }) + if i < 0 || slices.Contains(p.Movers[i].Published, ref) { + continue + } + in := builds.PublishedInput{Module: p.Movers[i].Module, Artifact: ref, Fingerprint: p.Movers[i].Fingerprint} + key := p.Pipeline + "\x00" + ref + "\x00" + in.Fingerprint + s.publishedMu.Lock() + a, asked := s.publishedRead[key] + s.publishedMu.Unlock() + if !asked { + var err error + if a, err = s.AskPublished(in); err != nil { + s.logf("%s: whether %s at %s is published is not read, so %s waits for its publish event: %v", p.Pipeline, ref, in.Fingerprint, mover, err) + continue + } + s.publishedMu.Lock() + if s.publishedRead == nil { + s.publishedRead = map[string]builds.PublishedAnswer{} + } + s.publishedRead[key] = a + s.publishedMu.Unlock() + } + if !a.Published { + continue + } + p.Movers[i].Published = append(p.Movers[i].Published, ref) + added = true + s.logf("%s: %s's wait on %s is resolved from the build seat's published (at %s), not from an event: placed from the published record (H-A-326)", p.Pipeline, mover, ref, a.Version) + } + return added } // holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view @@ -365,6 +416,9 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog // A mover's steps wait for the publish of each mover it builds on (§746, H-A-83 (2)): placed once each is heard, // ended not asked when one's build ended without passing. due, waiting, failed := ready(*p, m.Module, entries) + if !due && failed == "" && s.readPublished(p, m.Module, waiting) { + due, waiting, failed = ready(*p, m.Module, entries) + } if !due && failed == "" { s.logf("%s: %s waits for %s before its build", p.Pipeline, m.Module, strings.Join(waiting, ", ")) waits = append(waits, m) diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 7de591004..5c6ef7684 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -106,6 +106,8 @@ func TestManifest(t *testing.T) { // A-102 (§22, §24): the accepted kinds the runner puts its steps on. "seat:mesh-builds.step", "seat:mesh-wants.step", "seat:node-build-go.check.step", "seat:node-build-node.check.step", "seat:node-build-image.check.step", + // H-A-328: whether a package a mover waits on is published, read when the wait is written. + "seat:mesh-builds.published", // RN22 (§86, §415, A-207): a recovery's steps on the build seats' stream delivery. "seat:node-build-go.delivery.step", "seat:node-build-node.delivery.step", "seat:node-build-image.delivery.step", // A-231: the queues of the build seats it asks, check for the steps and delivery for a recovery's. diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index d25b519de..ebe2436af 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -78,6 +78,7 @@ "seat:mesh-module-assigner.assignments", "seat:mesh-grants.grants-for", "seat:mesh-builds.step", + "seat:mesh-builds.published", "seat:mesh-wants.step", "seat:node-build-go.check.step", "seat:node-build-node.check.step",