Read every pipeline key form by one parse; the assigner moves an again pipeline's tiers (H-A-485)

The assigner took a tier progress for `pipeline.<branch>.<revision>` only (two dots), so
the progress of a pipeline started again at the head, `<branch>.<revision>.again.<n>`,
was never moved and read as an empty end: its later tiers were never wanted. One parse
of the runner's seat's key forms (mesh-delivery-sdk pipelinekey, over the contract's key
parsers) now reads a pipeline's key for the assigner's tier progress and the runner's
mover keys; a pipeline-ended whose key is of no form is refused loudly and nothing of
it is kept.

Builds: design 47 H2 H-A-485
This commit is contained in:
2026-11-22 18:54:40 +01:00
parent 5d3164b5b7
commit a5495bb15e
9 changed files with 210 additions and 27 deletions
@@ -17,6 +17,7 @@ import (
mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
@@ -166,6 +167,12 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" {
return false, nil
}
// The pipeline's key is read by the one parse of the runner's seat's key forms (design 47 H2 H-A-485): a key it does not
// know is refused loudly, and nothing of the pipeline is kept, never an empty tier progress.
if _, err := pipelinekey.Parse(p.Pipeline); err != nil {
e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: %v", p.Pipeline, err)
return false, nil
}
changed := false
var kept []tierMover
versioned := map[string]bool{}
@@ -96,6 +96,17 @@ func TestThePassedVersionIsKept(t *testing.T) {
ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}},
{"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)),
ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}},
// H-A-485: an again pipeline's end is kept as a take-in's; a key of no form the runner's seat defines is refused, nothing kept.
{"an again pipeline keeps its version", []mpr.PipelineEnded{func() mpr.PipelineEnded {
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
p.Pipeline = "main.5.again.2"
return p
}()}, &End{Pipeline: "main.5.again.2", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
{"a key of no form is refused", []mpr.PipelineEnded{func() mpr.PipelineEnded {
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
p.Pipeline = "main.5.later.2"
return p
}()}, nil},
{"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded {
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
p.Run = "r1"
@@ -109,6 +120,9 @@ func TestThePassedVersionIsKept(t *testing.T) {
}
}
got, kept := e.endOf("web", "main")
if ks, _ := e.st.keys(); c.name == "a key of no form is refused" && len(ks) != 0 {
t.Errorf("%s: the state written holds %v, want nothing (no empty tier progress)", c.name, ks)
}
switch {
case c.want == nil && kept:
t.Errorf("%s: kept %+v", c.name, got)
@@ -6,12 +6,14 @@ import (
"fmt"
"slices"
"sort"
"strconv"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/views"
@@ -58,10 +60,16 @@ type tierProgress struct {
// tierKey is the key of pipeline's tier progress; pipeline is the runner's key `<branch>.<source>`, its branch escaped.
func tierKey(pipeline string) string { return tierPrefix + pipeline }
// isTierKey says whether a key of the state written is a pipeline's tier progress: three parts, the first `pipeline` (a module's
// end has two).
// isTierKey says whether a key of the state written is a pipeline's tier progress: `pipeline.` and a pipeline's key of any form
// the runner's seat defines, read by the one parse (design 47 H2 H-A-485), so an again pipeline's progress is one as a take-in's
// is (a module's end, `<module>.<branch>`, is none).
func isTierKey(key string) bool {
return strings.HasPrefix(key, tierPrefix) && strings.Count(key, ".") == 2
rest, ok := strings.CutPrefix(key, tierPrefix)
if !ok {
return false
}
_, err := pipelinekey.Parse(rest)
return err == nil
}
// startTiers writes the tier progress of a passed pipeline, holding every passed mover, versioned or not, its first tier the
@@ -166,11 +174,11 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st
continue
}
pipeline := strings.TrimPrefix(key, tierPrefix)
branchEsc, number, _ := strings.Cut(pipeline, ".")
branch, err := keys.UnescapeName(branchEsc)
pk, err := pipelinekey.Parse(pipeline)
if err != nil {
continue
}
branch, number := pk.Branch, strconv.FormatInt(pk.Revision, 10)
for {
t, rev, ok := p.ends.tiersOf(pipeline)
if !ok {
@@ -28,7 +28,12 @@ import (
// tier-bound: the finding tier-late names it and its machine, pipeline-wanted says tier-late, and no consumer want is written.
func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
db, web := meshcatalogclient.SourcePrefix2+strings.Repeat("d", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64)
for _, runs := range []bool{true, false} {
// H-A-485: a pipeline started again at the head, keyed `<branch>.<revision>.again.<n>`, moves tier by tier as a take-in's does.
for _, c := range []struct {
runs bool
pipeline string
}{{true, trunkBranch + ".5"}, {false, trunkBranch + ".5"}, {true, trunkBranch + ".5.again.2"}, {false, trunkBranch + ".5.again.2"}} {
runs, pipeline := c.runs, c.pipeline
start := time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC)
now := start
wantsKV := &memKV{values: map[string][]byte{}}
@@ -41,7 +46,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
var said []meshwants.PipelineWanted
p.publishWants = func(event string, body any, _ string) error {
if b, ok := body.(meshwants.PipelineWantedEvent); ok {
if event != "pipeline-wanted."+trunkBranch+".5" {
if event != "pipeline-wanted."+pipeline {
t.Errorf("pipeline-wanted said as %s", event)
}
said = append(said, b)
@@ -65,7 +70,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
// The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216).
e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))
e.Source = 4
e.Source, e.Pipeline = 4, pipeline
if _, err := p.ends.take(e); err != nil {
t.Fatal(err)
}
@@ -76,7 +81,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
}
p.wants(context.Background())
if !wanted("g14", "db") || !wanted("ace", "db") || wanted("g14", "web") || wanted("ace", "web") {
t.Fatalf("runs %v: tier 1 alone is not what is wanted first: %v", runs, wantsKV.values)
t.Fatalf("%s runs %v: tier 1 alone is not what is wanted first: %v", pipeline, runs, wantsKV.values)
}
if runs {
raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: "db", Fingerprint: db})
@@ -86,15 +91,15 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
now = start.Add(time.Minute)
p.wants(context.Background())
if !wanted("g14", "web") || !wanted("ace", "web") {
t.Fatalf("the consumer's want is not written once its provider runs: %v", wantsKV.values)
t.Fatalf("%s: the consumer's want is not written once its provider runs: %v", pipeline, wantsKV.values)
}
raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web})
_ = p.w.hearRunning("running.g14.web", raw)
p.wants(context.Background())
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 {
t.Errorf("every tier runs: said %+v, findings %v", said, findings)
t.Errorf("%s: every tier runs: said %+v, findings %v", pipeline, said, findings)
}
if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok {
if _, _, ok := p.ends.tiersOf(pipeline); ok {
t.Error("the tier progress stands after every tier ran")
}
continue
@@ -107,7 +112,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
now = start.Add(30 * time.Minute)
p.wants(context.Background())
p.wants(context.Background())
f, ok := findings["delivery."+assign.KeyID(trunkBranch+".5")+".tier-late"]
f, ok := findings["delivery."+assign.KeyID(pipeline)+".tier-late"]
if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") {
t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings)
}
@@ -116,7 +121,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said)
}
// A tier progress kept before it held its source says the key's number as both (H-A-216).
p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
p.sayPipelineWanted(pipeline, trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 {
t.Errorf("a progress with no source: said %+v, want revision and source 5", said)
}
@@ -11,6 +11,7 @@ import (
"sync"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
"git.novox.be/novox/mesh-sdk/go/keys"
)
@@ -28,23 +29,20 @@ func StepsKey(pipeline, mover string) string {
// stepsPrefix is the start of every mover key of one pipeline.
func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") }
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a
// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`), an unfinished mover's own or one started again at
// the head (6 parts, its fourth `unfinished`, H-A-398, or `again`, H-A-479) or a publish's (7 parts) and one mover.
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.`, a pipeline's
// key of any form the seat defines (a take-in's, a run's, a publish's, an unfinished mover's own or one started again at the head,
// H-A-398, H-A-479), read by the one parse of the seat's key forms (design 47 H2 H-A-485), and one mover.
func IsStepsKey(key string) bool {
parts := strings.Split(key, ".")
if parts[0] != "steps" {
rest, ok := strings.CutPrefix(key, "steps.")
if !ok {
return false
}
switch len(parts) {
case 4, 7:
return true
case 5:
return parts[2] == runToken
case 6:
return parts[3] == unfinishedToken || parts[3] == againToken
i := strings.LastIndex(rest, ".")
if i <= 0 || i == len(rest)-1 {
return false
}
return false
_, err := pipelinekey.Parse(rest[:i])
return err == nil
}
// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by
@@ -519,6 +519,13 @@ func TestStartWithModulesStartsAFailedMoverAgainAtTheHead(t *testing.T) {
if !IsStepsKey(StepsKey("main.9.again.1", "app")) {
t.Error("a mover key of an again pipeline is not read as a mover's steps")
}
// H-A-485: the mover keys are read by the one parse of the seat's key forms; a pipeline key of no form (its n or its
// revision no number) is no mover's steps.
for _, k := range []string{"steps.main.9.again.x.app", "steps.main.r9.app", "steps.main.9.again.0.app"} {
if IsStepsKey(k) {
t.Errorf("%s, of no pipeline key form, is read as a mover's steps", k)
}
}
}
// H-A-479 addendum (2): a rerun of a step in an ended pipeline pins its revision again at the reopening (its pipeline-started
+1
View File
@@ -13,6 +13,7 @@ require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000
@@ -0,0 +1,100 @@
// Package pipelinekey reads the key of a pipeline in the seat mesh-pipeline-runner's state pipelines, in every key form the
// seat defines, through the seat's contract (design 47 H2 H-A-485): one parse, used by the runner, the assigner and the view, so
// a pipeline started again at the head (`<branch>.<revision>.again.<n>`, H-A-479) or an unfinished mover's own pipeline
// (`<branch>.<revision>.unfinished.<module>`, H-A-398) is read as a take-in's is. A key of no form the seat defines is refused
// with an error, never read as an empty one.
package pipelinekey
import (
"fmt"
"strconv"
"strings"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The key forms of a pipeline in the state pipelines (the seat's key forms, design 47 H §746, H-A-213 (2), H-A-398, H-A-479).
const (
FormRevision = "<branch>.<revision>"
FormRun = "<branch>.run.<run>"
FormPublished = "<branch>.<source>.published.<package>.<version>"
FormUnfinished = "<branch>.<revision>.unfinished.<module>"
FormAgain = "<branch>.<revision>.again.<n>"
)
// Key is a pipeline's key read: its form, its watched branch (unescaped), and the parts its form names: the revision (a
// published form's source), the run, the package and version (escaped, as the key holds them), the unfinished mover's
// module (unescaped) and the again form's n.
type Key struct {
Form string
Branch string
Revision int64
Run string
Package string
Version string
Module string
N int
}
// Parse reads key as a pipeline's key of one of the seat's forms; any other key is an error naming the forms.
func Parse(key string) (Key, error) {
parts := strings.Split(key, ".")
var k Key
var branch, revision, n, module string
ok := false
switch {
case len(parts) == 2:
branch, revision, ok = seat.ParsePipelinesByBranchRevisionKey(key)
k.Form = FormRevision
case len(parts) == 3 && parts[1] == "run":
branch, k.Run, ok = seat.ParsePipelinesByBranchRunRunKey(key)
k.Form = FormRun
case len(parts) == 4 && parts[2] == "unfinished":
branch, revision, module, ok = seat.ParsePipelinesByBranchRevisionUnfinishedModuleKey(key)
k.Form = FormUnfinished
case len(parts) == 4 && parts[2] == "again":
branch, revision, n, ok = seat.ParsePipelinesByBranchRevisionAgainNKey(key)
k.Form = FormAgain
case len(parts) == 5 && parts[2] == "published":
branch, revision, k.Package, k.Version, ok = seat.ParsePipelinesByBranchSourcePublishedPackageVersionKey(key)
k.Form = FormPublished
}
if !ok {
return Key{}, refused(key, "it is of no form the seat defines")
}
var err error
if k.Branch, err = keys.UnescapeName(branch); err != nil || k.Branch == "" {
return Key{}, refused(key, "its branch does not read")
}
if k.Form != FormRun {
if k.Revision, err = strconv.ParseInt(revision, 10, 64); err != nil || k.Revision <= 0 {
return Key{}, refused(key, "its revision is no number above 0")
}
}
switch k.Form {
case FormRun:
if k.Run == "" {
return Key{}, refused(key, "it names no run")
}
case FormUnfinished:
if k.Module, err = keys.UnescapeName(module); err != nil || k.Module == "" {
return Key{}, refused(key, "its module does not read")
}
case FormAgain:
if k.N, err = strconv.Atoi(n); err != nil || k.N < 1 {
return Key{}, refused(key, "its n is no number from 1")
}
case FormPublished:
if k.Package == "" || k.Version == "" {
return Key{}, refused(key, "it names no package or no version")
}
}
return k, nil
}
// refused is the error of a key Parse does not read.
func refused(key, why string) error {
return fmt.Errorf("%q is no key of a pipeline of the seat %s (%s): %s", key, seat.Seat,
strings.Join([]string{FormRevision, FormRun, FormPublished, FormUnfinished, FormAgain}, ", "), why)
}
@@ -0,0 +1,43 @@
package pipelinekey
import (
"strings"
"testing"
)
// H-A-485: every key form of a pipeline the seat mesh-pipeline-runner defines is read, the again form and the unfinished form as
// a take-in's, its branch and module unescaped; a key of no form, or whose parts do not read, is refused naming why.
func TestEveryPipelineKeyFormIsReadAndAnyOtherRefused(t *testing.T) {
for _, c := range []struct {
key string
want Key
why string
}{
{key: "main.265", want: Key{Form: FormRevision, Branch: "main", Revision: 265}},
{key: "main.265.again.2", want: Key{Form: FormAgain, Branch: "main", Revision: 265, N: 2}},
{key: "feat=2fx.12.unfinished.mesh-tools", want: Key{Form: FormUnfinished, Branch: "feat/x", Revision: 12, Module: "mesh-tools"}},
{key: "main.run.r7", want: Key{Form: FormRun, Branch: "main", Run: "r7"}},
{key: "main.40.published.lib.0=2e1=2e2", want: Key{Form: FormPublished, Branch: "main", Revision: 40, Package: "lib", Version: "0=2e1=2e2"}},
{key: "main", why: "no form"},
{key: "main.265.again", why: "no form"},
{key: "main.265.later.2", why: "no form"},
{key: "main.265.again.0", why: "its n"},
{key: "main.265.again.x", why: "its n"},
{key: "main.r1", why: "its revision"},
{key: "main.0", why: "its revision"},
{key: "ma=zz.5", why: "its branch"},
{key: "main.5.unfinished.=zz", why: "its module"},
{key: "main.5.6.7.8.9", why: "no form"},
} {
got, err := Parse(c.key)
if c.why != "" {
if err == nil || !strings.Contains(err.Error(), c.why) || !strings.Contains(err.Error(), FormAgain) {
t.Errorf("%s: read %+v (%v); want it refused, %s, naming the forms", c.key, got, err, c.why)
}
continue
}
if err != nil || got != c.want {
t.Errorf("%s: read %+v (%v); want %+v", c.key, got, err, c.want)
}
}
}