diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 19e9c98a5..0e6ecb14e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -179,6 +179,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) } starter.Left = hearer.LeftInPlace // a same-source re-pin's superseded steps leave their runs (H-A-284) starter.Revision = widener.Revision // a kept mover is placed from its kept revision (H-A-286) + // A mover's wait on a package is read first: a package published before the wait places it at once (H-A-326, H-A-328). + starter.AskPublished = publishedOf(boundedCalls(calls.Call, readBound, time.After, logger.Printf)) hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } hearer.AskPinHeld = func(key, module string, pins map[string]string) { if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go new file mode 100644 index 000000000..1b4fbdbb2 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/published.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "fmt" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// publishedOf asks the seat mesh-builds' verb published through call whether a package built at a fingerprint is published +// (design 47 H-A-326 (1), H-A-328), asked when a mover's wait on it is written. +func publishedOf(call func(string, any) (json.RawMessage, error)) func(builds.PublishedInput) (builds.PublishedAnswer, error) { + return func(in builds.PublishedInput) (builds.PublishedAnswer, error) { + raw, err := call("seat:"+builds.Seat+"."+builds.VerbPublished, in) + if err != nil { + return builds.PublishedAnswer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return builds.PublishedAnswer{}, fmt.Errorf("%s answered: %s", builds.VerbPublished, said) + } + var a builds.PublishedAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return builds.PublishedAnswer{}, fmt.Errorf("%s answered what does not read: %w", builds.VerbPublished, err) + } + return a, nil + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 4d4aeff0c..75edef4d0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "fmt" "slices" "strings" "testing" @@ -141,6 +142,39 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { t.Errorf("tool's step-ended = %+v, want not asked naming builder", ev.Result) } } + + // H-A-326 (1), (3), H-A-328: a package published before its importer's wait was written (by an earlier pipeline) is read of + // the build seat when the wait is written, matched by its mover's fingerprint, and the importer is placed with no event; + // the journal says the wait was resolved from the build seat's published, and a retried placing asks it once. + mem2 := memStore{} + var asked []builds.PublishedInput + var journal []string + s2 := &Starter{Pipelines: memCreated{mem2}, Store: mem2, Say: func(string, any) error { return nil }, Steps: &stepAsks{}, Name: "the-runner", + Registration: func(string) (catalog.Registration, bool) { return reg, true }, + Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }, + AskPublished: func(in builds.PublishedInput) (builds.PublishedAnswer, error) { + asked = append(asked, in) + return builds.PublishedAnswer{Published: in.Artifact == "mesh-sdk:go" && in.Fingerprint == "src2:00", Version: "0.2.0"}, nil + }} + if n, err := s2.TakenIn(ev, view); n != 1 || err != nil { + t.Fatalf("the take-in started %d (%v)", n, err) + } + p, _, _ = mem2.Get("main.7") + if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || !slices.Contains(sdk.Published, "mesh-sdk:go") { + t.Errorf("with mesh-sdk:go published before the wait: app placed %v, tool placed %v, mesh-sdk %+v; want app placed with no event", placed(p, "app"), placed(p, "tool"), sdk) + } + if !slices.ContainsFunc(journal, func(l string) bool { + return strings.Contains(l, "app's wait on mesh-sdk:go") && strings.Contains(l, "placed from the published record") + }) { + t.Errorf("the journal %q does not say app's wait was resolved from the build seat's published", journal) + } + before2 := len(asked) + if _, err := s2.Waiting(mem2, "main", view); err != nil { + t.Fatal(err) + } + if len(asked) != before2 { + t.Errorf("a placing again asked published %d more times, want each package asked once", len(asked)-before2) + } } // Design 47 G §270, A-326 (H §746, H-A-83 (2)): what a build module's new version is built from is built by its running diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/repin.go b/modules/mesh-delivery-pipeline/internal/pipeline/repin.go index a13a850ff..93a2dcc18 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/repin.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/repin.go @@ -177,6 +177,9 @@ func (s *Starter) placeAgain(p *seat.Pipeline, m seat.Mover, half Half, now time return nil, nil, true } due, waiting, failed := ready(*p, m.Module, half.Entries) + if !due && failed == "" && s.readPublished(p, m.Module, waiting) { + due, waiting, failed = ready(*p, m.Module, half.Entries) + } if !due && failed == "" { s.logf("%s: %s waits for %s before its build", p.Pipeline, m.Module, strings.Join(waiting, ", ")) return nil, nil, false diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 4469a8099..62926dc07 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" @@ -199,6 +200,56 @@ type Starter struct { // registration-missing (H-A-262); Observer is the runner's own module name. nil raises none. Findings Findings Observer string + // AskPublished asks the seat mesh-builds' verb published whether a package built at a fingerprint is published (H-A-326 + // (1), H-A-328): read when a mover's wait on it is written, so a publish made before the wait places the mover without + // any event; nil reads nothing, the publish event alone placing it. + AskPublished func(in builds.PublishedInput) (builds.PublishedAnswer, error) + + publishedMu sync.Mutex + publishedRead map[string]builds.PublishedAnswer +} + +// readPublished asks, once per pipeline, package and fingerprint, the seat mesh-builds whether each package mover waits on +// in p is published already (H-A-326 (1), H-A-328): a wait is a read first and an event second. Each package answered +// published goes into its From mover's published in p, written with the placing, and the journal says the wait resolved +// from the build seat's published, apart from one a package-published places (H-A-326 (3)). The answer is kept for the +// placing's compare-and-set retries, so no retry asks again. It says whether it added any. +func (s *Starter) readPublished(p *seat.Pipeline, mover string, waiting []string) bool { + if s.AskPublished == nil { + return false + } + added := false + for _, ref := range waiting { + i := slices.IndexFunc(p.Movers, func(m seat.Mover) bool { return m.Module == moduleOf(ref) }) + if i < 0 || slices.Contains(p.Movers[i].Published, ref) { + continue + } + in := builds.PublishedInput{Module: p.Movers[i].Module, Artifact: ref, Fingerprint: p.Movers[i].Fingerprint} + key := p.Pipeline + "\x00" + ref + "\x00" + in.Fingerprint + s.publishedMu.Lock() + a, asked := s.publishedRead[key] + s.publishedMu.Unlock() + if !asked { + var err error + if a, err = s.AskPublished(in); err != nil { + s.logf("%s: whether %s at %s is published is not read, so %s waits for its publish event: %v", p.Pipeline, ref, in.Fingerprint, mover, err) + continue + } + s.publishedMu.Lock() + if s.publishedRead == nil { + s.publishedRead = map[string]builds.PublishedAnswer{} + } + s.publishedRead[key] = a + s.publishedMu.Unlock() + } + if !a.Published { + continue + } + p.Movers[i].Published = append(p.Movers[i].Published, ref) + added = true + s.logf("%s: %s's wait on %s is resolved from the build seat's published (at %s), not from an event: placed from the published record (H-A-326)", p.Pipeline, mover, ref, a.Version) + } + return added } // holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view @@ -365,6 +416,9 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog // A mover's steps wait for the publish of each mover it builds on (§746, H-A-83 (2)): placed once each is heard, // ended not asked when one's build ended without passing. due, waiting, failed := ready(*p, m.Module, entries) + if !due && failed == "" && s.readPublished(p, m.Module, waiting) { + due, waiting, failed = ready(*p, m.Module, entries) + } if !due && failed == "" { s.logf("%s: %s waits for %s before its build", p.Pipeline, m.Module, strings.Join(waiting, ", ")) waits = append(waits, m) diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 7de591004..5c6ef7684 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -106,6 +106,8 @@ func TestManifest(t *testing.T) { // A-102 (§22, §24): the accepted kinds the runner puts its steps on. "seat:mesh-builds.step", "seat:mesh-wants.step", "seat:node-build-go.check.step", "seat:node-build-node.check.step", "seat:node-build-image.check.step", + // H-A-328: whether a package a mover waits on is published, read when the wait is written. + "seat:mesh-builds.published", // RN22 (§86, §415, A-207): a recovery's steps on the build seats' stream delivery. "seat:node-build-go.delivery.step", "seat:node-build-node.delivery.step", "seat:node-build-image.delivery.step", // A-231: the queues of the build seats it asks, check for the steps and delivery for a recovery's. diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index d25b519de..ebe2436af 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -78,6 +78,7 @@ "seat:mesh-module-assigner.assignments", "seat:mesh-grants.grants-for", "seat:mesh-builds.step", + "seat:mesh-builds.published", "seat:mesh-wants.step", "seat:node-build-go.check.step", "seat:node-build-node.check.step",