From 5d3164b5b726b097833f6f0205f7190086d2103d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 18:49:52 +0100 Subject: [PATCH 1/2] Builder: take a released version the catalog holds at the start; a refused tag waits the pause (H-A-484) A version-released whose tag the seat git's holder refused was handed back at once, spent its five deliveries in seconds and was dropped by the bus: its reservation stood and its publish item asked again every 30 s for ever. At the start the builder now holds every built, unreleased reservation against the catalog's verb versions and takes a released one in as its version-released would (tag, release, bound and item ended, findings cleared); a refused tag is handed back with the publish pause. The tag check itself is the forge module's, already accepting a nested package path on the catalog's trunk. Builds: design 47 H2 H-A-484, H-A-484 addendum --- .../cmd/mesh-delivery-builder/main.go | 10 ++- .../cmd/mesh-delivery-builder/reserve.go | 87 ++++++++++++++----- .../cmd/mesh-delivery-builder/reserve_test.go | 38 +++++++- 3 files changed, 111 insertions(+), 24 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 5fe1094f3..3f0e8e7b4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -298,7 +298,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1)) + pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1)) + reserved.pause = publishSet.Pause // a version-released whose tag the forge did not write waits the pause (H-A-484 addendum) releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept) if err != nil { return err @@ -322,7 +323,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)). republish.Ended = items.Ended pq.take = items.take - reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) }) + // At the start, a built reservation the catalog holds released is taken in as its version-released would (H-A-484 addendum), + // before the publish items of the rest are asked. + reasksKV.Ready(func(jetstream.KeyValue) { + reserved.releasedAtStart(ctx) + items.AskAllDone(ctx) + }) wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept) acts.Republish = republish.Republish acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 53e946a10..5ffcead8b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -59,6 +59,9 @@ type reservations struct { withdraw func(module, fingerprint, why string) // call asks a seat's verb: the git seat's tag (design 47 G §370). call toolCaller + // pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands + // it back at once. + pause time.Duration // rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none. rebuild func(ctx context.Context, module, fp, why string) error // say says package-published on the seat mesh-builds; nil says none. @@ -310,32 +313,20 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err) break } - if err := rv.tagged(ctx, ev); err != nil { + if refused, err := rv.release(ctx, ev); err != nil { rv.logf("%v; heard again", err) - if nakErr := m.Nak(); nakErr != nil { + // A tag the seat git's holder did not write is handed back with the publish pause, so the deliveries the consumer + // allows span the holder's outage (design 47 H2 H-A-484 addendum). + nak := m.Nak + if refused && rv.pause > 0 { + nak = func() error { return m.NakWithDelay(rv.pause) } + } + if nakErr := nak(); nakErr != nil { rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) } givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf) return true } - gone, err := rv.r.Released(ctx, ev.Module, ev.Version) - if err != nil { - rv.logf("%v; heard again", err) - if nakErr := m.Nak(); nakErr != nil { - rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) - } - givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf) - return true - } - for _, k := range gone { - rv.dropBound(ctx, k) - } - if rv.released != nil { - rv.released(ctx, ev.Module, ev.Version) - } - rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module) - rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module) - rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) default: return false } @@ -345,6 +336,62 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { return true } +// release takes a released version in as the catalog's version-released says it (design 47 G §369, §370): its tag written, its +// reservation released, its bound and publish item ended and its findings cleared. refused says the tag was not written (the +// seat git did not write it, or the reservation could not be read to tag it); the reservation then stands. +func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) (refused bool, err error) { + if err := rv.tagged(ctx, ev); err != nil { + return true, err + } + gone, err := rv.r.Released(ctx, ev.Module, ev.Version) + if err != nil { + return false, err + } + for _, k := range gone { + rv.dropBound(ctx, k) + } + if rv.released != nil { + rv.released(ctx, ev.Module, ev.Version) + } + rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module) + rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module) + rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) + return false, nil +} + +// releasedAtStart takes in, at the builder's start, every built, unreleased reservation whose version the catalog's verb versions +// holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one +// said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended. +// A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released. +func (rv *reservations) releasedAtStart(ctx context.Context) { + all, err := rv.r.All(ctx) + if err != nil { + rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err) + return + } + read := map[string][]string{} + for _, v := range builtOf(all) { + if versions.NotStanding(v, true) != "" { + continue + } + released, ok := read[v.Module] + if !ok { + if released, err = rv.r.ReleasedOf(ctx, v.Module); err != nil { + rv.logf("%s: the catalog's released versions cannot be read at the start, so its built reservations wait for its next version-released: %v", v.Module, err) + } + read[v.Module] = released + } + if !slices.Contains(released, v.Version) { + continue + } + if _, err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil { + rv.logf("%s %s is released in the catalog and not taken in at the start: %v", v.Module, v.Version, err) + continue + } + rv.logf("%s %s is released in the catalog: taken in at the start, as its version-released", v.Module, v.Version) + } +} + // published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told // before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told // nothing. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 874e41f9f..076f709b7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -164,14 +164,15 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { ctx := context.Background() var tags []git.TagInput refuse := "" - var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796) + var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796) + var catalog []meshcatalog.VersionEntry // the released versions the catalog's verb versions answers call := func(addr string, body any) (json.RawMessage, error) { switch addr { case "seat:git.tags": return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true}) case "seat:git.tag": default: - return json.Marshal(meshcatalog.VersionsAnswer{}) + return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog}) } tags = append(tags, body.(git.TagInput)) return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse}) @@ -203,11 +204,44 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { if m := release("pkg", "0.1.0"); !m.naked || m.acked { t.Fatalf("a tag the forge did not write: naked %v acked %v; want the release heard again", m.naked, m.acked) } + // Design 47 H2 H-A-484 addendum: with a publish pause set, a refused tag hands the version-released back after the pause, so + // the deliveries its consumer allows span the holder's outage. + rv.pause = 30 * time.Second + if m := release("pkg", "0.1.0"); m.delayed != 30*time.Second || m.naked || m.acked { + t.Fatalf("a tag the forge did not write, with the pause: delayed %v naked %v acked %v; want handed back after 30s", m.delayed, m.naked, m.acked) + } + rv.pause = 0 if all, _ := rv.r.All(ctx); len(all) != 1 { t.Fatalf("the reservation went with an unwritten tag: %v", all) } + // Design 47 H2 H-A-484 addendum: at the builder's start, a built reservation the catalog holds released is tagged and released + // as its version-released would (a version-released the bus gave up on), and one the catalog does not hold, or not built, + // stands. refuse = "" tags = nil + var ended []string + rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) } + rv.releasedAtStart(ctx) // built nothing yet: nothing taken + if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 { + t.Fatalf("at the start with no build done: reservations %v, tags %+v; want the reservation standing, no tag", all, tags) + } + if _, err := rv.r.Built(ctx, "pkg", "0.1.0", "pkg:lib", "sha256:aa", nil); err != nil { + t.Fatal(err) + } + rv.releasedAtStart(ctx) // the catalog holds none released: it stands + if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 { + t.Fatalf("at the start with nothing released: reservations %v, tags %+v; want the reservation standing", all, tags) + } + catalog = []meshcatalog.VersionEntry{{Module: "pkg", Version: "0.1.0"}} + rv.releasedAtStart(ctx) + if all, _ := rv.r.All(ctx); len(all) != 0 || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) || !slices.Equal(ended, []string{"pkg 0.1.0"}) { + t.Fatalf("at the start with 0.1.0 released: reservations %v, tags %+v, items ended %v; want it tagged, released and its item ended", all, tags, ended) + } + catalog, rv.released = nil, nil + if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // the version is reserved again for the rows below + t.Fatal(err) + } + tags = nil if m := release("pkg", "0.1.0"); !m.acked || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) { t.Fatalf("the release: acked %v, tags %+v; want go/v0.1.0 at c0ffee of o/r", m.acked, tags) } -- 2.54.0 From a5495bb15e27b19f06190b82b1b16e98e96c0b91 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 18:54:40 +0100 Subject: [PATCH 2/2] Read every pipeline key form by one parse; the assigner moves an again pipeline's tiers (H-A-485) The assigner took a tier progress for `pipeline..` only (two dots), so the progress of a pipeline started again at the head, `..again.`, was never moved and read as an empty end: its later tiers were never wanted. One parse of the runner's seat's key forms (mesh-delivery-sdk pipelinekey, over the contract's key parsers) now reads a pipeline's key for the assigner's tier progress and the runner's mover keys; a pipeline-ended whose key is of no form is refused loudly and nothing of it is kept. Builds: design 47 H2 H-A-485 --- .../cmd/mesh-delivery-assigner/pipelines.go | 7 ++ .../mesh-delivery-assigner/pipelines_test.go | 14 +++ .../cmd/mesh-delivery-assigner/tiers.go | 18 +++- .../cmd/mesh-delivery-assigner/tiers_test.go | 23 ++-- .../internal/pipeline/moverkeys.go | 24 ++--- .../internal/pipeline/verbs_test.go | 7 ++ modules/mesh-delivery-sdk/go.mod | 1 + .../pipelinekey/pipelinekey.go | 100 ++++++++++++++++++ .../pipelinekey/pipelinekey_test.go | 43 ++++++++ 9 files changed, 210 insertions(+), 27 deletions(-) create mode 100644 modules/mesh-delivery-sdk/pipelinekey/pipelinekey.go create mode 100644 modules/mesh-delivery-sdk/pipelinekey/pipelinekey_test.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index f07068e58..65f614222 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -17,6 +17,7 @@ import ( mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" @@ -166,6 +167,12 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" { return false, nil } + // The pipeline's key is read by the one parse of the runner's seat's key forms (design 47 H2 H-A-485): a key it does not + // know is refused loudly, and nothing of the pipeline is kept, never an empty tier progress. + if _, err := pipelinekey.Parse(p.Pipeline); err != nil { + e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: %v", p.Pipeline, err) + return false, nil + } changed := false var kept []tierMover versioned := map[string]bool{} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index 90c80e9bb..fb10310eb 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -96,6 +96,17 @@ func TestThePassedVersionIsKept(t *testing.T) { ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}}, {"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)), ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}}, + // H-A-485: an again pipeline's end is kept as a take-in's; a key of no form the runner's seat defines is refused, nothing kept. + {"an again pipeline keeps its version", []mpr.PipelineEnded{func() mpr.PipelineEnded { + p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok)) + p.Pipeline = "main.5.again.2" + return p + }()}, &End{Pipeline: "main.5.again.2", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}}, + {"a key of no form is refused", []mpr.PipelineEnded{func() mpr.PipelineEnded { + p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok)) + p.Pipeline = "main.5.later.2" + return p + }()}, nil}, {"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)) p.Run = "r1" @@ -109,6 +120,9 @@ func TestThePassedVersionIsKept(t *testing.T) { } } got, kept := e.endOf("web", "main") + if ks, _ := e.st.keys(); c.name == "a key of no form is refused" && len(ks) != 0 { + t.Errorf("%s: the state written holds %v, want nothing (no empty tier progress)", c.name, ks) + } switch { case c.want == nil && kept: t.Errorf("%s: kept %+v", c.name, got) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go index 4e321cb11..6422d3ce7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go @@ -6,12 +6,14 @@ import ( "fmt" "slices" "sort" + "strconv" "strings" "time" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/views" @@ -58,10 +60,16 @@ type tierProgress struct { // tierKey is the key of pipeline's tier progress; pipeline is the runner's key `.`, its branch escaped. func tierKey(pipeline string) string { return tierPrefix + pipeline } -// isTierKey says whether a key of the state written is a pipeline's tier progress: three parts, the first `pipeline` (a module's -// end has two). +// isTierKey says whether a key of the state written is a pipeline's tier progress: `pipeline.` and a pipeline's key of any form +// the runner's seat defines, read by the one parse (design 47 H2 H-A-485), so an again pipeline's progress is one as a take-in's +// is (a module's end, `.`, is none). func isTierKey(key string) bool { - return strings.HasPrefix(key, tierPrefix) && strings.Count(key, ".") == 2 + rest, ok := strings.CutPrefix(key, tierPrefix) + if !ok { + return false + } + _, err := pipelinekey.Parse(rest) + return err == nil } // startTiers writes the tier progress of a passed pipeline, holding every passed mover, versioned or not, its first tier the @@ -166,11 +174,11 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st continue } pipeline := strings.TrimPrefix(key, tierPrefix) - branchEsc, number, _ := strings.Cut(pipeline, ".") - branch, err := keys.UnescapeName(branchEsc) + pk, err := pipelinekey.Parse(pipeline) if err != nil { continue } + branch, number := pk.Branch, strconv.FormatInt(pk.Revision, 10) for { t, rev, ok := p.ends.tiersOf(pipeline) if !ok { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go index 842a0c36c..8bdcd70b4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go @@ -28,7 +28,12 @@ import ( // tier-bound: the finding tier-late names it and its machine, pipeline-wanted says tier-late, and no consumer want is written. func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { db, web := meshcatalogclient.SourcePrefix2+strings.Repeat("d", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64) - for _, runs := range []bool{true, false} { + // H-A-485: a pipeline started again at the head, keyed `..again.`, moves tier by tier as a take-in's does. + for _, c := range []struct { + runs bool + pipeline string + }{{true, trunkBranch + ".5"}, {false, trunkBranch + ".5"}, {true, trunkBranch + ".5.again.2"}, {false, trunkBranch + ".5.again.2"}} { + runs, pipeline := c.runs, c.pipeline start := time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC) now := start wantsKV := &memKV{values: map[string][]byte{}} @@ -41,7 +46,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { var said []meshwants.PipelineWanted p.publishWants = func(event string, body any, _ string) error { if b, ok := body.(meshwants.PipelineWantedEvent); ok { - if event != "pipeline-wanted."+trunkBranch+".5" { + if event != "pipeline-wanted."+pipeline { t.Errorf("pipeline-wanted said as %s", event) } said = append(said, b) @@ -65,7 +70,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { // The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216). e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed), mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed)) - e.Source = 4 + e.Source, e.Pipeline = 4, pipeline if _, err := p.ends.take(e); err != nil { t.Fatal(err) } @@ -76,7 +81,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { } p.wants(context.Background()) if !wanted("g14", "db") || !wanted("ace", "db") || wanted("g14", "web") || wanted("ace", "web") { - t.Fatalf("runs %v: tier 1 alone is not what is wanted first: %v", runs, wantsKV.values) + t.Fatalf("%s runs %v: tier 1 alone is not what is wanted first: %v", pipeline, runs, wantsKV.values) } if runs { raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: "db", Fingerprint: db}) @@ -86,15 +91,15 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { now = start.Add(time.Minute) p.wants(context.Background()) if !wanted("g14", "web") || !wanted("ace", "web") { - t.Fatalf("the consumer's want is not written once its provider runs: %v", wantsKV.values) + t.Fatalf("%s: the consumer's want is not written once its provider runs: %v", pipeline, wantsKV.values) } raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web}) _ = p.w.hearRunning("running.g14.web", raw) p.wants(context.Background()) if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 { - t.Errorf("every tier runs: said %+v, findings %v", said, findings) + t.Errorf("%s: every tier runs: said %+v, findings %v", pipeline, said, findings) } - if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok { + if _, _, ok := p.ends.tiersOf(pipeline); ok { t.Error("the tier progress stands after every tier ran") } continue @@ -107,7 +112,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { now = start.Add(30 * time.Minute) p.wants(context.Background()) p.wants(context.Background()) - f, ok := findings["delivery."+assign.KeyID(trunkBranch+".5")+".tier-late"] + f, ok := findings["delivery."+assign.KeyID(pipeline)+".tier-late"] if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") { t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings) } @@ -116,7 +121,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said) } // A tier progress kept before it held its source says the key's number as both (H-A-216). - p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil) + p.sayPipelineWanted(pipeline, trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil) if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 { t.Errorf("a progress with no source: said %+v, want revision and source 5", said) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go index d6b4280f9..8d6d32d30 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go @@ -11,6 +11,7 @@ import ( "sync" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey" "git.novox.be/novox/mesh-sdk/go/keys" ) @@ -28,23 +29,20 @@ func StepsKey(pipeline, mover string) string { // stepsPrefix is the start of every mover key of one pipeline. func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") } -// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a -// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`), an unfinished mover's own or one started again at -// the head (6 parts, its fourth `unfinished`, H-A-398, or `again`, H-A-479) or a publish's (7 parts) and one mover. +// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.`, a pipeline's +// key of any form the seat defines (a take-in's, a run's, a publish's, an unfinished mover's own or one started again at the head, +// H-A-398, H-A-479), read by the one parse of the seat's key forms (design 47 H2 H-A-485), and one mover. func IsStepsKey(key string) bool { - parts := strings.Split(key, ".") - if parts[0] != "steps" { + rest, ok := strings.CutPrefix(key, "steps.") + if !ok { return false } - switch len(parts) { - case 4, 7: - return true - case 5: - return parts[2] == runToken - case 6: - return parts[3] == unfinishedToken || parts[3] == againToken + i := strings.LastIndex(rest, ".") + if i <= 0 || i == len(rest)-1 { + return false } - return false + _, err := pipelinekey.Parse(rest[:i]) + return err == nil } // RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index 0a29c03b1..53954de59 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -519,6 +519,13 @@ func TestStartWithModulesStartsAFailedMoverAgainAtTheHead(t *testing.T) { if !IsStepsKey(StepsKey("main.9.again.1", "app")) { t.Error("a mover key of an again pipeline is not read as a mover's steps") } + // H-A-485: the mover keys are read by the one parse of the seat's key forms; a pipeline key of no form (its n or its + // revision no number) is no mover's steps. + for _, k := range []string{"steps.main.9.again.x.app", "steps.main.r9.app", "steps.main.9.again.0.app"} { + if IsStepsKey(k) { + t.Errorf("%s, of no pipeline key form, is read as a mover's steps", k) + } + } } // H-A-479 addendum (2): a rerun of a step in an ended pipeline pins its revision again at the reopening (its pipeline-started diff --git a/modules/mesh-delivery-sdk/go.mod b/modules/mesh-delivery-sdk/go.mod index 43d0cfa07..03f521e9f 100644 --- a/modules/mesh-delivery-sdk/go.mod +++ b/modules/mesh-delivery-sdk/go.mod @@ -13,6 +13,7 @@ require ( git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-sdk/pipelinekey/pipelinekey.go b/modules/mesh-delivery-sdk/pipelinekey/pipelinekey.go new file mode 100644 index 000000000..10e32ad3b --- /dev/null +++ b/modules/mesh-delivery-sdk/pipelinekey/pipelinekey.go @@ -0,0 +1,100 @@ +// Package pipelinekey reads the key of a pipeline in the seat mesh-pipeline-runner's state pipelines, in every key form the +// seat defines, through the seat's contract (design 47 H2 H-A-485): one parse, used by the runner, the assigner and the view, so +// a pipeline started again at the head (`..again.`, H-A-479) or an unfinished mover's own pipeline +// (`..unfinished.`, H-A-398) is read as a take-in's is. A key of no form the seat defines is refused +// with an error, never read as an empty one. +package pipelinekey + +import ( + "fmt" + "strconv" + "strings" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The key forms of a pipeline in the state pipelines (the seat's key forms, design 47 H §746, H-A-213 (2), H-A-398, H-A-479). +const ( + FormRevision = "." + FormRun = ".run." + FormPublished = "..published.." + FormUnfinished = "..unfinished." + FormAgain = "..again." +) + +// Key is a pipeline's key read: its form, its watched branch (unescaped), and the parts its form names: the revision (a +// published form's source), the run, the package and version (escaped, as the key holds them), the unfinished mover's +// module (unescaped) and the again form's n. +type Key struct { + Form string + Branch string + Revision int64 + Run string + Package string + Version string + Module string + N int +} + +// Parse reads key as a pipeline's key of one of the seat's forms; any other key is an error naming the forms. +func Parse(key string) (Key, error) { + parts := strings.Split(key, ".") + var k Key + var branch, revision, n, module string + ok := false + switch { + case len(parts) == 2: + branch, revision, ok = seat.ParsePipelinesByBranchRevisionKey(key) + k.Form = FormRevision + case len(parts) == 3 && parts[1] == "run": + branch, k.Run, ok = seat.ParsePipelinesByBranchRunRunKey(key) + k.Form = FormRun + case len(parts) == 4 && parts[2] == "unfinished": + branch, revision, module, ok = seat.ParsePipelinesByBranchRevisionUnfinishedModuleKey(key) + k.Form = FormUnfinished + case len(parts) == 4 && parts[2] == "again": + branch, revision, n, ok = seat.ParsePipelinesByBranchRevisionAgainNKey(key) + k.Form = FormAgain + case len(parts) == 5 && parts[2] == "published": + branch, revision, k.Package, k.Version, ok = seat.ParsePipelinesByBranchSourcePublishedPackageVersionKey(key) + k.Form = FormPublished + } + if !ok { + return Key{}, refused(key, "it is of no form the seat defines") + } + var err error + if k.Branch, err = keys.UnescapeName(branch); err != nil || k.Branch == "" { + return Key{}, refused(key, "its branch does not read") + } + if k.Form != FormRun { + if k.Revision, err = strconv.ParseInt(revision, 10, 64); err != nil || k.Revision <= 0 { + return Key{}, refused(key, "its revision is no number above 0") + } + } + switch k.Form { + case FormRun: + if k.Run == "" { + return Key{}, refused(key, "it names no run") + } + case FormUnfinished: + if k.Module, err = keys.UnescapeName(module); err != nil || k.Module == "" { + return Key{}, refused(key, "its module does not read") + } + case FormAgain: + if k.N, err = strconv.Atoi(n); err != nil || k.N < 1 { + return Key{}, refused(key, "its n is no number from 1") + } + case FormPublished: + if k.Package == "" || k.Version == "" { + return Key{}, refused(key, "it names no package or no version") + } + } + return k, nil +} + +// refused is the error of a key Parse does not read. +func refused(key, why string) error { + return fmt.Errorf("%q is no key of a pipeline of the seat %s (%s): %s", key, seat.Seat, + strings.Join([]string{FormRevision, FormRun, FormPublished, FormUnfinished, FormAgain}, ", "), why) +} diff --git a/modules/mesh-delivery-sdk/pipelinekey/pipelinekey_test.go b/modules/mesh-delivery-sdk/pipelinekey/pipelinekey_test.go new file mode 100644 index 000000000..2e3162508 --- /dev/null +++ b/modules/mesh-delivery-sdk/pipelinekey/pipelinekey_test.go @@ -0,0 +1,43 @@ +package pipelinekey + +import ( + "strings" + "testing" +) + +// H-A-485: every key form of a pipeline the seat mesh-pipeline-runner defines is read, the again form and the unfinished form as +// a take-in's, its branch and module unescaped; a key of no form, or whose parts do not read, is refused naming why. +func TestEveryPipelineKeyFormIsReadAndAnyOtherRefused(t *testing.T) { + for _, c := range []struct { + key string + want Key + why string + }{ + {key: "main.265", want: Key{Form: FormRevision, Branch: "main", Revision: 265}}, + {key: "main.265.again.2", want: Key{Form: FormAgain, Branch: "main", Revision: 265, N: 2}}, + {key: "feat=2fx.12.unfinished.mesh-tools", want: Key{Form: FormUnfinished, Branch: "feat/x", Revision: 12, Module: "mesh-tools"}}, + {key: "main.run.r7", want: Key{Form: FormRun, Branch: "main", Run: "r7"}}, + {key: "main.40.published.lib.0=2e1=2e2", want: Key{Form: FormPublished, Branch: "main", Revision: 40, Package: "lib", Version: "0=2e1=2e2"}}, + {key: "main", why: "no form"}, + {key: "main.265.again", why: "no form"}, + {key: "main.265.later.2", why: "no form"}, + {key: "main.265.again.0", why: "its n"}, + {key: "main.265.again.x", why: "its n"}, + {key: "main.r1", why: "its revision"}, + {key: "main.0", why: "its revision"}, + {key: "ma=zz.5", why: "its branch"}, + {key: "main.5.unfinished.=zz", why: "its module"}, + {key: "main.5.6.7.8.9", why: "no form"}, + } { + got, err := Parse(c.key) + if c.why != "" { + if err == nil || !strings.Contains(err.Error(), c.why) || !strings.Contains(err.Error(), FormAgain) { + t.Errorf("%s: read %+v (%v); want it refused, %s, naming the forms", c.key, got, err, c.why) + } + continue + } + if err != nil || got != c.want { + t.Errorf("%s: read %+v (%v); want %+v", c.key, got, err, c.want) + } + } +} -- 2.54.0