From b6702196b88945a0d93d5034f8ce9b74e7097188 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 23:35:51 +0100 Subject: [PATCH] H-A-504 addendum assigner: a want's run is its pipeline's sequence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runOf took the pipeline key's last segment, an again run's number, so node-tools 0.1.115 of main.320.again.1 (run "1") was refused on every node under run "2" of main.265.again.2, an older pipeline. The assigner now keeps the sequence pipeline-ended says in its end record and the want rule writes it as the want's run; the compare-and-set and the node-engine's compare stay as they are. A pipeline-ended without seq is refused in the journal and keeps nothing; an end kept before this change, without seq, writes no want and says so once, naming the runner's end-again of its pipeline as the recovery. Builds: design 47 H2 §813 (1), H-A-504 corrected, H-A-504 addendum, H-A-504 second addendum --- .../mesh-delivery-assigner/graphkept_test.go | 36 +++++++++++----- .../integration_test.go | 2 +- .../cmd/mesh-delivery-assigner/pass.go | 15 +++---- .../cmd/mesh-delivery-assigner/pipelines.go | 42 ++++++++++++++++-- .../mesh-delivery-assigner/pipelines_test.go | 43 ++++++++++++++----- .../internal/assign/wants.go | 7 +++ 6 files changed, 111 insertions(+), 34 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go index b8078d174..f90e3ef05 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go @@ -96,21 +96,22 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) { if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest { t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events)) } - // Design 47 H2 §813 (1), H-A-416: a want carries its run, its pipeline's source in decimal, or the newest source held for a - // want of no pipeline; it is written by compare-and-set refusing a lower run, and one that differs only by its run stands. + // Design 47 H2 §813 (1), H-A-504 corrected and its addendum: a want of a pipeline carries its run, the pipeline's sequence the + // want rule set; a want of no pipeline the newest source held. It is written by compare-and-set refusing a lower run, and + // one that differs only by its run stands. t.Run("a want's run", func(t *testing.T) { wkey, _ := mwc.WantKey("g14", "db") stored := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[wkey]); return w } - p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1}) - if w := stored(); w.Run != "12" || w.Pipeline != "main.12" { - t.Fatalf("the want of pipeline main.12 is %+v, want run 12", w) + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1, Run: "212"}) + if w := stored(); w.Run != "212" || w.Pipeline != "main.12" { + t.Fatalf("the want of pipeline main.12 is %+v, want run 212", w) } puts := kv.puts - p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1}) - if w := stored(); w.Run != "12" || w.Version != "0.1.2" || kv.puts != puts { + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1, Run: "211"}) + if w := stored(); w.Run != "212" || w.Version != "0.1.2" || kv.puts != puts { t.Errorf("a want of a lower run replaced %+v (%d writes)", w, kv.puts-puts) } - p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1}) + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1, Run: "212"}) if kv.puts != puts { t.Error("a want that stands was written again") } @@ -119,10 +120,23 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) { if w, _ := mwc.DecodeWant(kv.values[skey]); w.Run != "9" { t.Errorf("a put's want is %+v, want the newest source held, 9", w) } - // a placement's build after the delivery of run 12: the newest source held for it is the want's own, never refused + // a placement's build after the delivery of run 212: the newest source held for it is the want's own, never refused p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.0"}) - if w := stored(); w.Run != "12" || w.Version != "0.1.0" { - t.Errorf("a placement's want after run 12 is %+v, want it written at run 12", w) + if w := stored(); w.Run != "212" || w.Version != "0.1.0" { + t.Errorf("a placement's want after run 212 is %+v, want it written at run 212", w) + } + // A want written with an again run's number ("2") is replaced by any sequence; a lower sequence is refused. + nkey, _ := mwc.WantKey("novox", "node-tools") + atNovox := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[nkey]); return w } + p.putWant(ctx, "novox", "node-tools", mwc.Want{Fp: fpTest, Version: "0.1.96", Pipeline: "main.265.again.2", Rank: 1, Run: "2"}) + p.putWant(ctx, "novox", "node-tools", mwc.Want{Fp: fpTest, Version: "0.1.115", Pipeline: "main.320.again.1", Rank: 1, Run: "270"}) + if w := atNovox(); w.Run != "270" || w.Version != "0.1.115" { + t.Errorf("seq 270 after run 2 left %+v, want 0.1.115 at run 270", w) + } + puts = kv.puts + p.putWant(ctx, "novox", "node-tools", mwc.Want{Fp: fpTest, Version: "0.1.114", Pipeline: "main.319", Rank: 1, Run: "269"}) + if w := atNovox(); w.Run != "270" || kv.puts != puts { + t.Errorf("seq 269 after 270 replaced %+v (%d writes)", w, kv.puts-puts) } }) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 8358c76d8..0531f2932 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -599,7 +599,7 @@ func (b *bed) ended(t *testing.T, source int64, outcome string, movers ...moverA if outcome != mpr.PipelineEndedOutcomePassed { step.Outcome = mpr.StepResultOutcomeFailed } - e := mpr.PipelineEnded{V: 1, Pipeline: pipeline, Branch: trunkBranch, Source: source, Outcome: outcome} + e := mpr.PipelineEnded{V: 1, Pipeline: pipeline, Branch: trunkBranch, Source: source, Seq: 200 + source, Outcome: outcome} for _, m := range movers { e.Movers = append(e.Movers, mpr.MoverEnded{Module: m.module, Fingerprint: m.fp, Version: m.version, Tier: m.tier, Steps: []mpr.StepResult{step}}) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index fcd40fdf1..6dc77a0fb 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -312,15 +312,14 @@ func (p *passer) putWant(ctx context.Context, node, module string, want mwc.Want } } -// runOf is the run a want carries (design 47 H2 §813 (1); H-A-355 (1), H-A-416): the source revision of the pipeline that -// released it, `.`, in decimal; for a want of no pipeline (a put, a placement's build, a removal) the newest -// source the assigner holds for it: the trunk's source of its kept assignment graph revision, or the run of the want written -// (held) when that is newer, so a person's put or placement after a delivery is never refused as older; "" when it holds none. +// runOf is the run a want carries (design 47 H2 §813 (1); H-A-355 (1), H-A-504 corrected and its addendum): the sequence of +// the pipeline that released it, as its pipeline-ended said it, set on the want by the want rule; for a want of no pipeline (a +// put, a placement's build, a removal) the newest source the assigner holds for it: the trunk's source of its kept assignment +// graph revision, or the run of the want written (held) when that is newer, so a person's put or placement after a delivery is +// never refused as older; "" when it holds none. func (p *passer) runOf(want mwc.Want, held *mwc.Want) string { - if i := strings.LastIndexByte(want.Pipeline, '.'); i >= 0 { - if n, err := strconv.ParseInt(want.Pipeline[i+1:], 10, 64); err == nil && n > 0 { - return strconv.FormatInt(n, 10) - } + if want.Run != "" { + return want.Run } var newest int64 if p.graph != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index c1883405f..87d43f3a3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -44,8 +44,11 @@ const pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipe type End struct { Pipeline string `json:"pipeline"` Source int64 `json:"source"` - Fp string `json:"fp"` - Version string `json:"version"` + // Seq is the pipeline's sequence its pipeline-ended said, written as the run of its wants (design 47 H2 §813 (1), H-A-504 + // addendum). + Seq int64 `json:"seq,omitempty"` + Fp string `json:"fp"` + Version string `json:"version"` // Rank is the mover's tier in the pipeline's delivery order, from 1 (§746, §750). Rank int64 `json:"rank"` } @@ -141,6 +144,8 @@ type pipelineEnds struct { // firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design // 47 G §14). firsts map[string]string + // noSeq is, by module and pipeline, the ends kept without a sequence already said (H-A-504 second addendum). + noSeq map[string]bool // runEnded is told a run on request's pipeline-ended, its group `run-`, to drop the run's candidate (A-235, H-A-58); nil // drops nothing. runEnded func(group string) error @@ -178,6 +183,13 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: %v", p.Pipeline, err) return false, nil } + // Its sequence is its wants' run (design 47 H2 §813 (1), H-A-504 addendum): an end without one, of an older runner, is + // refused loudly and writes no want. + if p.Seq <= 0 { + e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: it carries no seq, the run its "+ + "wants would carry (a runner older than H-A-504's addendum); the runner's end-again of %s, once it says seq, keeps them", p.Pipeline, p.Pipeline) + return false, nil + } changed := false var kept []tierMover versioned := map[string]bool{} @@ -219,7 +231,7 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { break // an older version, or this one again } } - value, err := json.Marshal(End{Pipeline: p.Pipeline, Source: p.Source, Fp: m.Fingerprint, Version: m.Version, Rank: max(m.Tier, 1)}) + value, err := json.Marshal(End{Pipeline: p.Pipeline, Source: p.Source, Seq: p.Seq, Fp: m.Fingerprint, Version: m.Version, Rank: max(m.Tier, 1)}) if err != nil { return changed, err } @@ -236,6 +248,22 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { return changed, nil } +// noSeqOnce says, once per module and pipeline, that the end kept for module from pipeline carries no sequence and writes +// no want, naming the recovery (H-A-504 second addendum). +func (e *pipelineEnds) noSeqOnce(module, pipeline string) { + e.mu.Lock() + if e.noSeq == nil { + e.noSeq = map[string]bool{} + } + said := e.noSeq[module+" "+pipeline] + e.noSeq[module+" "+pipeline] = true + e.mu.Unlock() + if !said && e.logf != nil { + e.logf("no want of %s is written: its end kept from %s carries no seq, the run its want would carry (kept before "+ + "H-A-504's addendum); the runner's end-again of %s, or %s's next pipeline, writes it", module, pipeline, pipeline, module) + } +} + // endOf is the end kept for module on branch: the newest version released there whose pipeline passed. func (e *pipelineEnds) endOf(module, branch string) (End, bool) { if e == nil { @@ -257,7 +285,13 @@ func (e *pipelineEnds) target(module, branch string) (assign.Target, bool) { if !ok || !e.tierWritten(end) { return assign.Target{}, false } - return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank}, true + if end.Seq <= 0 { + // An end kept before pipeline-ended carried its sequence has no run for its wants (design 47 H2 §813 (1), H-A-504 + // second addendum): it writes no want, said once. + e.noSeqOnce(module, end.Pipeline) + return assign.Target{}, false + } + return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank, Seq: end.Seq}, true } // hearPipeline takes a pipeline-ended: a passed one's released versions are kept (take), and a run on request's drops the diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index aff50e807..f870a6d70 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -68,9 +68,9 @@ func mover(module, fp, version string, tier int64, outcome string) mpr.MoverEnde Steps: []mpr.StepResult{{Step: "unit", Status: mpr.StepResultStatusRequired, Outcome: outcome, Module: module}}} } -// ended is the pipeline-ended of the take-in of source on branch, its movers given. +// ended is the pipeline-ended of the take-in of source on branch, its movers given, its sequence 200 + source. func ended(branch string, source int64, outcome string, movers ...mpr.MoverEnded) mpr.PipelineEnded { - return mpr.PipelineEnded{V: 1, Branch: branch, Source: source, EndCount: 1, Outcome: outcome, Movers: movers, + return mpr.PipelineEnded{V: 1, Branch: branch, Source: source, Seq: 200 + source, EndCount: 1, Outcome: outcome, Movers: movers, Pipeline: fmt.Sprintf("%s.%d", branch, source)} } @@ -86,28 +86,34 @@ func TestThePassedVersionIsKept(t *testing.T) { want *End // nil: none kept for web on main }{ {"a passed pipeline keeps its version", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))}, - &End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}}, + &End{Pipeline: "main.5", Source: 5, Seq: 205, Fp: "fp1", Version: "0.1.2", Rank: 2}}, {"nothing to run counts as passed", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, nothing))}, - &End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 1}}, + &End{Pipeline: "main.5", Source: 5, Seq: 205, Fp: "fp1", Version: "0.1.2", Rank: 1}}, {"a failed pipeline keeps nothing, a passed mover in it neither", []mpr.PipelineEnded{ended("main", 5, fail, mover("web", "fp1", "0.1.2", 1, ok), mover("db", "fp9", "0.3.0", 1, failed))}, nil}, {"a mover whose required step failed keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, failed))}, nil}, {"no version released keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "", 1, ok))}, nil}, {"an older end heard after a newer one keeps the newer", []mpr.PipelineEnded{ended("main", 6, pass, mover("web", "fp2", "0.1.3", 1, ok)), - ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}}, + ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Seq: 206, Fp: "fp2", Version: "0.1.3", Rank: 1}}, {"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)), - ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}}, + ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Seq: 206, Fp: "fp2", Version: "0.1.10", Rank: 1}}, // H-A-485: an again pipeline's end is kept as a take-in's; a key of no form the runner's seat defines is refused, nothing kept. {"an again pipeline keeps its version", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok)) p.Pipeline = "main.5.again.2" return p - }()}, &End{Pipeline: "main.5.again.2", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}}, + }()}, &End{Pipeline: "main.5.again.2", Source: 5, Seq: 205, Fp: "fp1", Version: "0.1.2", Rank: 2}}, {"a key of no form is refused", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok)) p.Pipeline = "main.5.later.2" return p }()}, nil}, + // H-A-504 addendum: an end without its sequence, of an older runner, is refused and keeps nothing. + {"an end without seq keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { + p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)) + p.Seq = 0 + return p + }()}, nil}, {"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)) p.Run = "r1" @@ -206,7 +212,7 @@ func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) { if !passed { continue } - if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 { + if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 || w.Run != "205" { t.Errorf("%s: the want on %s = %+v (%v)", c.name, node, w, err) } } @@ -235,8 +241,25 @@ func TestARegistrationNamesTheDefaultBranch(t *testing.T) { if woken == 0 || endsKept() != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch { t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", endsKept(), e.branchOf("web", "")) } - if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" { - t.Errorf("the default branch's target = %+v %v", tg, ok) + if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" || tg.Seq != 205 { + t.Errorf("the default branch's target = %+v %v, want fp2 at seq 205", tg, ok) + } + // H-A-504 second addendum: an end kept without a sequence (before pipeline-ended said it) writes no want, said once. + raw, rev, _ := st.get(endKey("web", "beta")) + var old End + _ = json.Unmarshal(raw, &old) + old.Seq = 0 + raw, _ = json.Marshal(old) + _ = st.swap(endKey("web", "beta"), raw, rev) + var said []string + e.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) } + for range 2 { + if tg, ok := e.target("web", ""); ok { + t.Errorf("an end kept without seq is the target %+v", tg) + } + } + if len(said) != 1 || !strings.Contains(said[0], "beta.5") || !strings.Contains(said[0], "end-again") { + t.Errorf("an end kept without seq said %q, want once, naming the pipeline and end-again", said) } e.registered(mpcat.Registration{Module: "web"}, true) if endsKept() != 0 { diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 6e4aea7e8..99fa6f5b7 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -3,6 +3,7 @@ package assign import ( "fmt" "slices" + "strconv" "strings" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" @@ -28,6 +29,9 @@ type Target struct { Version string Pipeline string Rank int64 + // Seq is the pipeline's sequence, the want's run in decimal (design 47 H2 §813 (1), H-A-504 addendum); 0 for a placement's + // build. + Seq int64 } // Module is what the want rule reads of one module: the nodes it is assigned to, the build each node's want names, the puts @@ -123,6 +127,9 @@ func Wants(m Module, wants map[string]mwc.Want) Outcome { continue } next := mwc.Want{Fp: t.Fp, Version: t.Version, Pipeline: t.Pipeline, Rank: t.Rank, Self: m.selfOn(node)} + if t.Seq > 0 { + next.Run = strconv.FormatInt(t.Seq, 10) + } if had && !before.Removed && !before.Put && before.Fp == t.Fp { // The answers the want carries stay while its build does: a failed backup's (ADR 0539 §16), a held revert's, the yes. next.Backup, next.Words, next.Revert, next.Yes, next.Release = before.Backup, before.Words, before.Revert, before.Yes, before.Release