Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6a550f59f6 | ||
|
|
b6702196b8 | ||
|
|
f6aff0182c | ||
|
|
ed0b905767 | ||
|
|
28726acaaf | ||
|
|
7bacccf562 | ||
|
|
29c4673ad1 | ||
|
|
a8240f5549 | ||
|
|
5792e83bc6 | ||
|
|
678d8fa5b3 | ||
|
|
b2d9d1ac90 | ||
|
|
6ffddedfbf | ||
|
|
146b9b4ff8 | ||
|
|
1d0a40592b | ||
|
|
7be95905f2 | ||
|
|
04fe2e30fe | ||
|
|
52ad5af1e9 | ||
|
|
cd16358b21 | ||
|
|
0c2ead025d | ||
|
|
5ab9bd9acc | ||
|
|
dc33ae26bb | ||
|
|
a096397928 | ||
|
|
5548fff81f | ||
|
|
52fa062e92 | ||
|
|
ef32d60a5e | ||
|
|
cfd2afe39c | ||
|
|
21daeb8893 | ||
|
|
ae1bf2098d | ||
|
|
f09bd2668a | ||
|
|
82156f8128 | ||
|
|
ffa58e51c4 | ||
|
|
cc1034630f | ||
|
|
73cb9ba690 | ||
|
|
fcac1fc5d0 | ||
|
|
db899d8ad4 | ||
|
|
4ca6e478a6 | ||
|
|
564fd5e0bb | ||
|
|
dc6dfef6f0 | ||
|
|
8eee17a368 | ||
|
|
2b79ee1593 | ||
|
|
9931d4f900 |
@@ -154,6 +154,13 @@ func (b *builtEnds) madeOf(module, fp string) map[string]string {
|
||||
return parts
|
||||
}
|
||||
|
||||
// releasedOf is the version released of module's done build at fp, the release whose parts are what its build.ended made (design
|
||||
// 47 G A-421); false when no such release is heard.
|
||||
func (w *world) releasedOf(module, fp string) (string, bool) {
|
||||
k, ok := w.released.withParts(module, w.ends.madeOf(module, fp))
|
||||
return k.Version, ok && k.Version != ""
|
||||
}
|
||||
|
||||
// of is the version of module released from the source revision, with its parts' digests; false before it is heard.
|
||||
func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bool) {
|
||||
r.mu.Lock()
|
||||
|
||||
@@ -96,21 +96,22 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
|
||||
if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest {
|
||||
t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events))
|
||||
}
|
||||
// Design 47 H2 §813 (1), H-A-416: a want carries its run, its pipeline's source in decimal, or the newest source held for a
|
||||
// want of no pipeline; it is written by compare-and-set refusing a lower run, and one that differs only by its run stands.
|
||||
// Design 47 H2 §813 (1), H-A-504 corrected and its addendum: a want of a pipeline carries its run, the pipeline's sequence the
|
||||
// want rule set; a want of no pipeline the newest source held. It is written by compare-and-set refusing a lower run, and
|
||||
// one that differs only by its run stands.
|
||||
t.Run("a want's run", func(t *testing.T) {
|
||||
wkey, _ := mwc.WantKey("g14", "db")
|
||||
stored := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[wkey]); return w }
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
|
||||
if w := stored(); w.Run != "12" || w.Pipeline != "main.12" {
|
||||
t.Fatalf("the want of pipeline main.12 is %+v, want run 12", w)
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1, Run: "212"})
|
||||
if w := stored(); w.Run != "212" || w.Pipeline != "main.12" {
|
||||
t.Fatalf("the want of pipeline main.12 is %+v, want run 212", w)
|
||||
}
|
||||
puts := kv.puts
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1})
|
||||
if w := stored(); w.Run != "12" || w.Version != "0.1.2" || kv.puts != puts {
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1, Run: "211"})
|
||||
if w := stored(); w.Run != "212" || w.Version != "0.1.2" || kv.puts != puts {
|
||||
t.Errorf("a want of a lower run replaced %+v (%d writes)", w, kv.puts-puts)
|
||||
}
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1, Run: "212"})
|
||||
if kv.puts != puts {
|
||||
t.Error("a want that stands was written again")
|
||||
}
|
||||
@@ -119,10 +120,23 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
|
||||
if w, _ := mwc.DecodeWant(kv.values[skey]); w.Run != "9" {
|
||||
t.Errorf("a put's want is %+v, want the newest source held, 9", w)
|
||||
}
|
||||
// a placement's build after the delivery of run 12: the newest source held for it is the want's own, never refused
|
||||
// a placement's build after the delivery of run 212: the newest source held for it is the want's own, never refused
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.0"})
|
||||
if w := stored(); w.Run != "12" || w.Version != "0.1.0" {
|
||||
t.Errorf("a placement's want after run 12 is %+v, want it written at run 12", w)
|
||||
if w := stored(); w.Run != "212" || w.Version != "0.1.0" {
|
||||
t.Errorf("a placement's want after run 212 is %+v, want it written at run 212", w)
|
||||
}
|
||||
// A want written with an again run's number ("2") is replaced by any sequence; a lower sequence is refused.
|
||||
nkey, _ := mwc.WantKey("novox", "node-tools")
|
||||
atNovox := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[nkey]); return w }
|
||||
p.putWant(ctx, "novox", "node-tools", mwc.Want{Fp: fpTest, Version: "0.1.96", Pipeline: "main.265.again.2", Rank: 1, Run: "2"})
|
||||
p.putWant(ctx, "novox", "node-tools", mwc.Want{Fp: fpTest, Version: "0.1.115", Pipeline: "main.320.again.1", Rank: 1, Run: "270"})
|
||||
if w := atNovox(); w.Run != "270" || w.Version != "0.1.115" {
|
||||
t.Errorf("seq 270 after run 2 left %+v, want 0.1.115 at run 270", w)
|
||||
}
|
||||
puts = kv.puts
|
||||
p.putWant(ctx, "novox", "node-tools", mwc.Want{Fp: fpTest, Version: "0.1.114", Pipeline: "main.319", Rank: 1, Run: "269"})
|
||||
if w := atNovox(); w.Run != "270" || kv.puts != puts {
|
||||
t.Errorf("seq 269 after 270 replaced %+v (%d writes)", w, kv.puts-puts)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -599,7 +599,7 @@ func (b *bed) ended(t *testing.T, source int64, outcome string, movers ...moverA
|
||||
if outcome != mpr.PipelineEndedOutcomePassed {
|
||||
step.Outcome = mpr.StepResultOutcomeFailed
|
||||
}
|
||||
e := mpr.PipelineEnded{V: 1, Pipeline: pipeline, Branch: trunkBranch, Source: source, Outcome: outcome}
|
||||
e := mpr.PipelineEnded{V: 1, Pipeline: pipeline, Branch: trunkBranch, Source: source, Seq: 200 + source, Outcome: outcome}
|
||||
for _, m := range movers {
|
||||
e.Movers = append(e.Movers, mpr.MoverEnded{Module: m.module, Fingerprint: m.fp, Version: m.version, Tier: m.tier, Steps: []mpr.StepResult{step}})
|
||||
}
|
||||
|
||||
@@ -656,6 +656,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
// once the first registrations are written (§95, §353).
|
||||
p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf)
|
||||
p.ends.trunk = w.trunkName
|
||||
p.ends.releasedOf = w.releasedOf // a version released before its pipeline passed is taken at the pass (H-A-490)
|
||||
p.ends.runEnded = func(group string) error { return p.dropCandidate(ctx, group) }
|
||||
if err := hearEnds(hub, p.ends); err != nil {
|
||||
logger.Printf("a pipeline's end is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err)
|
||||
|
||||
@@ -312,15 +312,14 @@ func (p *passer) putWant(ctx context.Context, node, module string, want mwc.Want
|
||||
}
|
||||
}
|
||||
|
||||
// runOf is the run a want carries (design 47 H2 §813 (1); H-A-355 (1), H-A-416): the source revision of the pipeline that
|
||||
// released it, `<branch>.<source>`, in decimal; for a want of no pipeline (a put, a placement's build, a removal) the newest
|
||||
// source the assigner holds for it: the trunk's source of its kept assignment graph revision, or the run of the want written
|
||||
// (held) when that is newer, so a person's put or placement after a delivery is never refused as older; "" when it holds none.
|
||||
// runOf is the run a want carries (design 47 H2 §813 (1); H-A-355 (1), H-A-504 corrected and its addendum): the sequence of
|
||||
// the pipeline that released it, as its pipeline-ended said it, set on the want by the want rule; for a want of no pipeline (a
|
||||
// put, a placement's build, a removal) the newest source the assigner holds for it: the trunk's source of its kept assignment
|
||||
// graph revision, or the run of the want written (held) when that is newer, so a person's put or placement after a delivery is
|
||||
// never refused as older; "" when it holds none.
|
||||
func (p *passer) runOf(want mwc.Want, held *mwc.Want) string {
|
||||
if i := strings.LastIndexByte(want.Pipeline, '.'); i >= 0 {
|
||||
if n, err := strconv.ParseInt(want.Pipeline[i+1:], 10, 64); err == nil && n > 0 {
|
||||
return strconv.FormatInt(n, 10)
|
||||
}
|
||||
if want.Run != "" {
|
||||
return want.Run
|
||||
}
|
||||
var newest int64
|
||||
if p.graph != nil {
|
||||
|
||||
@@ -409,7 +409,12 @@ func putOfTheBus(t *testing.T) {
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
putModuleRaw(p.w, trunkBranch, "nats", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
|
||||
endDone(t, p.w, "nats", fp, mdigest)
|
||||
// The build and its release (H-A-490 addendum): the put's want names the version released at its parts.
|
||||
built, _ := json.Marshal(builds.BuildEnded{Module: "nats", Fingerprint: fp, Outcome: builds.BuildDone, Manifest: mdigest, Made: map[string]string{"program": "sha256:5"}})
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "nats", Source: 8, Version: "2.10.1", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "nats:program", Digest: "sha256:5"}}})
|
||||
if p.w.ends.put(built) != nil || p.w.released.put(rel) != nil {
|
||||
t.Fatal("the build or its release is not heard")
|
||||
}
|
||||
hkey, _ := nodeengineclient.HaveKey("novox", "nats")
|
||||
before, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
world(buckets.Have, hkey, before)
|
||||
@@ -489,7 +494,7 @@ func putOfTheBus(t *testing.T) {
|
||||
<-woke
|
||||
got = step()
|
||||
w, err := mwc.DecodeWant(wantsKV.values["want.novox.nats"])
|
||||
if got.Step != puts.Wanted || err != nil || w.Fp != fp || !w.Put || !w.Hold {
|
||||
if got.Step != puts.Wanted || err != nil || w.Fp != fp || w.Version != "2.10.1" || !w.Put || !w.Hold {
|
||||
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||
}
|
||||
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
|
||||
|
||||
@@ -44,8 +44,11 @@ const pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipe
|
||||
type End struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Source int64 `json:"source"`
|
||||
Fp string `json:"fp"`
|
||||
Version string `json:"version"`
|
||||
// Seq is the pipeline's sequence its pipeline-ended said, written as the run of its wants (design 47 H2 §813 (1), H-A-504
|
||||
// addendum).
|
||||
Seq int64 `json:"seq,omitempty"`
|
||||
Fp string `json:"fp"`
|
||||
Version string `json:"version"`
|
||||
// Rank is the mover's tier in the pipeline's delivery order, from 1 (§746, §750).
|
||||
Rank int64 `json:"rank"`
|
||||
}
|
||||
@@ -141,6 +144,8 @@ type pipelineEnds struct {
|
||||
// firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design
|
||||
// 47 G §14).
|
||||
firsts map[string]string
|
||||
// noSeq is, by module and pipeline, the ends kept without a sequence already said (H-A-504 second addendum).
|
||||
noSeq map[string]bool
|
||||
// runEnded is told a run on request's pipeline-ended, its group `run-<id>`, to drop the run's candidate (A-235, H-A-58); nil
|
||||
// drops nothing.
|
||||
runEnded func(group string) error
|
||||
@@ -153,6 +158,10 @@ type pipelineEnds struct {
|
||||
trunk func() string
|
||||
// now is the clock of the tier progress (tiers.go); nil is time.Now.
|
||||
now func() time.Time
|
||||
// releasedOf is the version released of module's build at fp, matched by the parts its build.ended made (world.releasedOf):
|
||||
// a mover whose version was released before its pipeline passed gets its want at the pass (design 47 H2 H-A-490). nil
|
||||
// matches none.
|
||||
releasedOf func(module, fp string) (string, bool)
|
||||
}
|
||||
|
||||
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
|
||||
@@ -162,7 +171,8 @@ func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipel
|
||||
// take keeps, for each mover of a passed pipeline that released a version, that version as the newest of its module on the
|
||||
// pipeline's branch, by compare-and-set on a newer version (design 47 H §750): an older version, or the same one again, writes
|
||||
// nothing. A passed mover with no version yet is said in the journal and taken when the runner says the end again naming it
|
||||
// (H-A-270); it holds its tier in the pipeline's tier progress meanwhile (H-A-277 (1)). It says whether any key changed.
|
||||
// (H-A-270); it holds its tier in the pipeline's tier progress meanwhile (H-A-277 (1)). A mover whose build's version was
|
||||
// released before the pass is taken at the pass with that version (H-A-490). It says whether any key changed.
|
||||
func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" {
|
||||
return false, nil
|
||||
@@ -173,6 +183,13 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: %v", p.Pipeline, err)
|
||||
return false, nil
|
||||
}
|
||||
// Its sequence is its wants' run (design 47 H2 §813 (1), H-A-504 addendum): an end without one, of an older runner, is
|
||||
// refused loudly and writes no want.
|
||||
if p.Seq <= 0 {
|
||||
e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: it carries no seq, the run its "+
|
||||
"wants would carry (a runner older than H-A-504's addendum); the runner's end-again of %s, once it says seq, keeps them", p.Pipeline, p.Pipeline)
|
||||
return false, nil
|
||||
}
|
||||
changed := false
|
||||
var kept []tierMover
|
||||
versioned := map[string]bool{}
|
||||
@@ -185,6 +202,14 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
if m.Fingerprint == "" || !moverPassed(m) {
|
||||
continue
|
||||
}
|
||||
if m.Version == "" && e.releasedOf != nil {
|
||||
// A version released before its pipeline passed (an again pipeline on a built, published key) is never written
|
||||
// on the mover, nor its end said again: it is judged at the pass, matched by its build's parts (H-A-490).
|
||||
if v, ok := e.releasedOf(m.Module, m.Fingerprint); ok {
|
||||
e.logf("%s: %s passed; its version %s was released before the pass, taken now", p.Pipeline, m.Module, v)
|
||||
m.Version = v
|
||||
}
|
||||
}
|
||||
if m.Version == "" {
|
||||
// The runner writes a mover's version at its release's version-released, and says this end again once it does
|
||||
// (H-A-270 (1), (2)): the skip is said, never silent.
|
||||
@@ -206,7 +231,7 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
break // an older version, or this one again
|
||||
}
|
||||
}
|
||||
value, err := json.Marshal(End{Pipeline: p.Pipeline, Source: p.Source, Fp: m.Fingerprint, Version: m.Version, Rank: max(m.Tier, 1)})
|
||||
value, err := json.Marshal(End{Pipeline: p.Pipeline, Source: p.Source, Seq: p.Seq, Fp: m.Fingerprint, Version: m.Version, Rank: max(m.Tier, 1)})
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
@@ -223,6 +248,22 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// noSeqOnce says, once per module and pipeline, that the end kept for module from pipeline carries no sequence and writes
|
||||
// no want, naming the recovery (H-A-504 second addendum).
|
||||
func (e *pipelineEnds) noSeqOnce(module, pipeline string) {
|
||||
e.mu.Lock()
|
||||
if e.noSeq == nil {
|
||||
e.noSeq = map[string]bool{}
|
||||
}
|
||||
said := e.noSeq[module+" "+pipeline]
|
||||
e.noSeq[module+" "+pipeline] = true
|
||||
e.mu.Unlock()
|
||||
if !said && e.logf != nil {
|
||||
e.logf("no want of %s is written: its end kept from %s carries no seq, the run its want would carry (kept before "+
|
||||
"H-A-504's addendum); the runner's end-again of %s, or %s's next pipeline, writes it", module, pipeline, pipeline, module)
|
||||
}
|
||||
}
|
||||
|
||||
// endOf is the end kept for module on branch: the newest version released there whose pipeline passed.
|
||||
func (e *pipelineEnds) endOf(module, branch string) (End, bool) {
|
||||
if e == nil {
|
||||
@@ -244,7 +285,13 @@ func (e *pipelineEnds) target(module, branch string) (assign.Target, bool) {
|
||||
if !ok || !e.tierWritten(end) {
|
||||
return assign.Target{}, false
|
||||
}
|
||||
return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank}, true
|
||||
if end.Seq <= 0 {
|
||||
// An end kept before pipeline-ended carried its sequence has no run for its wants (design 47 H2 §813 (1), H-A-504
|
||||
// second addendum): it writes no want, said once.
|
||||
e.noSeqOnce(module, end.Pipeline)
|
||||
return assign.Target{}, false
|
||||
}
|
||||
return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank, Seq: end.Seq}, true
|
||||
}
|
||||
|
||||
// hearPipeline takes a pipeline-ended: a passed one's released versions are kept (take), and a run on request's drops the
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -67,9 +68,9 @@ func mover(module, fp, version string, tier int64, outcome string) mpr.MoverEnde
|
||||
Steps: []mpr.StepResult{{Step: "unit", Status: mpr.StepResultStatusRequired, Outcome: outcome, Module: module}}}
|
||||
}
|
||||
|
||||
// ended is the pipeline-ended of the take-in of source on branch, its movers given.
|
||||
// ended is the pipeline-ended of the take-in of source on branch, its movers given, its sequence 200 + source.
|
||||
func ended(branch string, source int64, outcome string, movers ...mpr.MoverEnded) mpr.PipelineEnded {
|
||||
return mpr.PipelineEnded{V: 1, Branch: branch, Source: source, EndCount: 1, Outcome: outcome, Movers: movers,
|
||||
return mpr.PipelineEnded{V: 1, Branch: branch, Source: source, Seq: 200 + source, EndCount: 1, Outcome: outcome, Movers: movers,
|
||||
Pipeline: fmt.Sprintf("%s.%d", branch, source)}
|
||||
}
|
||||
|
||||
@@ -85,28 +86,34 @@ func TestThePassedVersionIsKept(t *testing.T) {
|
||||
want *End // nil: none kept for web on main
|
||||
}{
|
||||
{"a passed pipeline keeps its version", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))},
|
||||
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
&End{Pipeline: "main.5", Source: 5, Seq: 205, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
{"nothing to run counts as passed", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, nothing))},
|
||||
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 1}},
|
||||
&End{Pipeline: "main.5", Source: 5, Seq: 205, Fp: "fp1", Version: "0.1.2", Rank: 1}},
|
||||
{"a failed pipeline keeps nothing, a passed mover in it neither", []mpr.PipelineEnded{ended("main", 5, fail, mover("web", "fp1", "0.1.2", 1, ok),
|
||||
mover("db", "fp9", "0.3.0", 1, failed))}, nil},
|
||||
{"a mover whose required step failed keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, failed))}, nil},
|
||||
{"no version released keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "", 1, ok))}, nil},
|
||||
{"an older end heard after a newer one keeps the newer", []mpr.PipelineEnded{ended("main", 6, pass, mover("web", "fp2", "0.1.3", 1, ok)),
|
||||
ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}},
|
||||
ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Seq: 206, Fp: "fp2", Version: "0.1.3", Rank: 1}},
|
||||
{"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)),
|
||||
ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}},
|
||||
ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Seq: 206, Fp: "fp2", Version: "0.1.10", Rank: 1}},
|
||||
// H-A-485: an again pipeline's end is kept as a take-in's; a key of no form the runner's seat defines is refused, nothing kept.
|
||||
{"an again pipeline keeps its version", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.again.2"
|
||||
return p
|
||||
}()}, &End{Pipeline: "main.5.again.2", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
}()}, &End{Pipeline: "main.5.again.2", Source: 5, Seq: 205, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
{"a key of no form is refused", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.later.2"
|
||||
return p
|
||||
}()}, nil},
|
||||
// H-A-504 addendum: an end without its sequence, of an older runner, is refused and keeps nothing.
|
||||
{"an end without seq keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
|
||||
p.Seq = 0
|
||||
return p
|
||||
}()}, nil},
|
||||
{"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
|
||||
p.Run = "r1"
|
||||
@@ -156,10 +163,15 @@ func TestThePassedVersionIsKept(t *testing.T) {
|
||||
// A6/AC1, A5/AC1 (design 47 H §750): a pass writes, on every node the module is assigned to, the want naming the newest version
|
||||
// released on its branch whose pipeline passed, with the pipeline and the rank; with none passed it writes none. No grant,
|
||||
// canary, provider or composition is asked for. H-A-270 (1): a passed end whose mover names no version yet writes no want and
|
||||
// says so in the journal; the same end said again naming the version writes it.
|
||||
// says so in the journal; the same end said again naming the version writes it. H-A-490: a version released before its pipeline
|
||||
// passed (an again pipeline on a built, published key) gets its want at the pass, matched to the mover's build by its parts.
|
||||
func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
for _, passed := range []bool{false, true} {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
passed, releasedBefore bool
|
||||
}{{"no pipeline passed", false, false}, {"passed, then its version released", true, false}, {"released, then its pipeline passed", true, true}} {
|
||||
passed := c.passed
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
@@ -172,7 +184,16 @@ func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
var journal []string
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
|
||||
p.ends.trunk = p.w.trunkName
|
||||
if passed {
|
||||
p.ends.releasedOf = p.w.releasedOf
|
||||
if c.releasedBefore {
|
||||
// Built and released in an earlier pipeline that failed: the runner's again pipeline ends naming no version.
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "web", Source: 4, Version: "0.1.2", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "web:image", Digest: "sha256:7"}}})
|
||||
built, _ := json.Marshal(builds.BuildEnded{Module: "web", Fingerprint: fp, Outcome: builds.BuildDone, Made: map[string]string{"image": "sha256:7"}})
|
||||
if err := p.w.released.put(rel); err != nil || p.w.ends.put(built) != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
|
||||
} else if passed {
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
|
||||
p.wants(context.Background())
|
||||
if len(wantsKV.values) != 0 || !slices.Contains(journal, trunkBranch+".5: web passed with no version yet: taken again when its version is written") {
|
||||
@@ -185,14 +206,14 @@ func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
raw, got := wantsKV.values[key]
|
||||
if got != passed {
|
||||
t.Errorf("passed %v: a want on %s written = %v", passed, node, got)
|
||||
t.Errorf("%s: a want on %s written = %v", c.name, node, got)
|
||||
continue
|
||||
}
|
||||
if !passed {
|
||||
continue
|
||||
}
|
||||
if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 {
|
||||
t.Errorf("the want on %s = %+v (%v)", node, w, err)
|
||||
if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 || w.Run != "205" {
|
||||
t.Errorf("%s: the want on %s = %+v (%v)", c.name, node, w, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -220,8 +241,25 @@ func TestARegistrationNamesTheDefaultBranch(t *testing.T) {
|
||||
if woken == 0 || endsKept() != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch {
|
||||
t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", endsKept(), e.branchOf("web", ""))
|
||||
}
|
||||
if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" {
|
||||
t.Errorf("the default branch's target = %+v %v", tg, ok)
|
||||
if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" || tg.Seq != 205 {
|
||||
t.Errorf("the default branch's target = %+v %v, want fp2 at seq 205", tg, ok)
|
||||
}
|
||||
// H-A-504 second addendum: an end kept without a sequence (before pipeline-ended said it) writes no want, said once.
|
||||
raw, rev, _ := st.get(endKey("web", "beta"))
|
||||
var old End
|
||||
_ = json.Unmarshal(raw, &old)
|
||||
old.Seq = 0
|
||||
raw, _ = json.Marshal(old)
|
||||
_ = st.swap(endKey("web", "beta"), raw, rev)
|
||||
var said []string
|
||||
e.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
for range 2 {
|
||||
if tg, ok := e.target("web", ""); ok {
|
||||
t.Errorf("an end kept without seq is the target %+v", tg)
|
||||
}
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "beta.5") || !strings.Contains(said[0], "end-again") {
|
||||
t.Errorf("an end kept without seq said %q, want once, naming the pipeline and end-again", said)
|
||||
}
|
||||
e.registered(mpcat.Registration{Module: "web"}, true)
|
||||
if endsKept() != 0 {
|
||||
|
||||
@@ -181,6 +181,7 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
|
||||
if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state
|
||||
w.Build, w.BuildWords = e.Outcome, e.Error
|
||||
}
|
||||
w.Version, _ = p.w.releasedOf(module, put.FP) // matched by its build's parts, as the pass matches a mover's (H-A-490 addendum)
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
|
||||
@@ -34,6 +34,9 @@ type PutWorld struct {
|
||||
Build, BuildWords string
|
||||
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
|
||||
Newest string
|
||||
// Version is the version released at the put's fingerprint, matched by its build's parts as the pass matches a mover's;
|
||||
// "" when none is released (design 47 H2 H-A-490 addendum).
|
||||
Version string
|
||||
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
|
||||
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
|
||||
Composed *ComposedDigest
|
||||
@@ -87,7 +90,10 @@ func StepPut(w PutWorld) PutNext {
|
||||
if h := w.Have; h == nil || h.Fp != p.FP || h.Composed == "" {
|
||||
return say(puts.Refused, "%s runs %s at %s, and its have names no composition at that fingerprint, so the want cannot name what runs", w.Node, w.Module, fp)
|
||||
}
|
||||
p.Composed = w.Have.Composed
|
||||
if w.Version == "" {
|
||||
return say(puts.Refused, "no version released at %s: build or release it first", fp)
|
||||
}
|
||||
p.Composed, p.Version = w.Have.Composed, w.Version
|
||||
words := fmt.Sprintf("the want of %s on %s names %s, the build it runs now (composition %s), from its running statement; the builder's store is not asked",
|
||||
w.Module, w.Node, fp, short(p.Composed))
|
||||
if p.Hold {
|
||||
@@ -115,6 +121,14 @@ func StepPut(w PutWorld) PutNext {
|
||||
default:
|
||||
return say(puts.Build, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
|
||||
}
|
||||
if p.Step != puts.Wanted && p.Step != puts.Holding {
|
||||
// The want names the build's released version beside its fingerprint, which the node-engine composes from (H-A-490
|
||||
// addendum): a build with none released is refused, never wanted by its fingerprint alone.
|
||||
if w.Version == "" {
|
||||
return say(puts.Refused, "no version released at %s: build or release it first", fp)
|
||||
}
|
||||
p.Version = w.Version
|
||||
}
|
||||
if w.Composed == nil || !w.Granted {
|
||||
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
|
||||
if w.Composed != nil {
|
||||
@@ -212,9 +226,10 @@ func needsPerson(w PutWorld, d ComposedDigest) bool {
|
||||
return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot
|
||||
}
|
||||
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint, marked put, and held when it holds.
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint and the version released at it, marked put, and held when
|
||||
// it holds.
|
||||
func PutWant(p puts.Put) mwc.Want {
|
||||
return mwc.Want{Fp: p.FP, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
return mwc.Want{Fp: p.FP, Version: p.Version, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
}
|
||||
|
||||
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
|
||||
|
||||
@@ -3,6 +3,7 @@ package assign
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
@@ -28,6 +29,9 @@ type Target struct {
|
||||
Version string
|
||||
Pipeline string
|
||||
Rank int64
|
||||
// Seq is the pipeline's sequence, the want's run in decimal (design 47 H2 §813 (1), H-A-504 addendum); 0 for a placement's
|
||||
// build.
|
||||
Seq int64
|
||||
}
|
||||
|
||||
// Module is what the want rule reads of one module: the nodes it is assigned to, the build each node's want names, the puts
|
||||
@@ -105,7 +109,7 @@ func Wants(m Module, wants map[string]mwc.Want) Outcome {
|
||||
if !put && had && before.Put && before.Hold {
|
||||
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard; only a put
|
||||
// heard released gives it back to the rule.
|
||||
p, put = puts.Put{FP: before.Fp, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), Step: puts.Holding}, true
|
||||
p, put = puts.Put{FP: before.Fp, Version: before.Version, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), Step: puts.Holding}, true
|
||||
}
|
||||
if put {
|
||||
if want, owns := PutOwns(p); owns {
|
||||
@@ -123,6 +127,9 @@ func Wants(m Module, wants map[string]mwc.Want) Outcome {
|
||||
continue
|
||||
}
|
||||
next := mwc.Want{Fp: t.Fp, Version: t.Version, Pipeline: t.Pipeline, Rank: t.Rank, Self: m.selfOn(node)}
|
||||
if t.Seq > 0 {
|
||||
next.Run = strconv.FormatInt(t.Seq, 10)
|
||||
}
|
||||
if had && !before.Removed && !before.Put && before.Fp == t.Fp {
|
||||
// The answers the want carries stay while its build does: a failed backup's (ADR 0539 §16), a held revert's, the yes.
|
||||
next.Backup, next.Words, next.Revert, next.Yes, next.Release = before.Backup, before.Words, before.Revert, before.Yes, before.Release
|
||||
|
||||
@@ -92,7 +92,7 @@ func TestAWantNamesTheNewestPassedVersion(t *testing.T) {
|
||||
// want stands, marked put and held, and once released the rule writes the passed version again.
|
||||
func TestAPutOwnsItsNodesWant(t *testing.T) {
|
||||
m := released()
|
||||
put := puts.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||
put := puts.Put{FP: fNew, Version: "1.2.3", Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
step string
|
||||
@@ -100,7 +100,7 @@ func TestAPutOwnsItsNodesWant(t *testing.T) {
|
||||
want *mwc.Want // nil: nothing written on ace
|
||||
}{
|
||||
{"waiting for the operator", puts.Person, wantedOld(), nil},
|
||||
{"wanted", puts.Wanted, wantedOld(), &mwc.Want{Fp: fNew, Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||
{"wanted", puts.Wanted, wantedOld(), &mwc.Want{Fp: fNew, Version: "1.2.3", Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||
{"holding, written already", puts.Holding, map[string]mwc.Want{"ace": PutWant(put)}, nil},
|
||||
{"released: the rule again", puts.Released, map[string]mwc.Want{"ace": PutWant(put)}, &mwc.Want{Fp: fNew, Version: "1.2.3", Pipeline: "main.12", Rank: 2}},
|
||||
{"not heard: a held want at an older fingerprint stands", "", map[string]mwc.Want{"ace": {Fp: fOld, Put: true, Hold: true, Words: "put: back"}}, nil},
|
||||
@@ -144,7 +144,7 @@ func stepsOfAPut(t *testing.T) {
|
||||
running := nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
|
||||
bus := func() PutWorld {
|
||||
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: builds.BuildDone, Newest: fNew,
|
||||
Composed: &d, Granted: true, Have: &running, Now: now}
|
||||
Version: "1.2.3", Composed: &d, Granted: true, Have: &running, Now: now}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -172,6 +172,16 @@ func stepsOfAPut(t *testing.T) {
|
||||
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
|
||||
return w
|
||||
}, puts.Refused, "names no composition", false, false, false},
|
||||
// H-A-490 (addendum): a want names the build's released version, so a done build with none released is refused, and so
|
||||
// is the build the node runs now when no release names its parts.
|
||||
{"a done build with no released version: refused", func() PutWorld { w := bus(); w.Version = ""; return w }, puts.Refused, "no version released at", false, false, false},
|
||||
{"the build the node runs now, no version released: refused", func() PutWorld {
|
||||
w := bus()
|
||||
w.Build, w.Newest, w.Version = "", fpOf('c'), ""
|
||||
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
|
||||
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
|
||||
return w
|
||||
}, puts.Refused, "build or release it first", false, false, false},
|
||||
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = builds.BuildFailed, "go vet"; return w }, puts.Refused, "go vet", false, false, false},
|
||||
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, puts.Compose, "no composition of nats for novox", false, false, false},
|
||||
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, puts.Compose, "does not name novox", false, false, false},
|
||||
@@ -256,6 +266,9 @@ func stepsOfAPut(t *testing.T) {
|
||||
}
|
||||
for _, c := range cases {
|
||||
next := StepPut(c.world())
|
||||
if (next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) && c.world().Put.Step == puts.Asked && next.Put.Version != "1.2.3" {
|
||||
t.Errorf("put, %s: its want names version %q, want the build's released 1.2.3", c.name, next.Put.Version)
|
||||
}
|
||||
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
|
||||
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
|
||||
}
|
||||
|
||||
@@ -65,6 +65,9 @@ type Put struct {
|
||||
V int `json:"v"`
|
||||
// FP is the fingerprint put, of the new form.
|
||||
FP string `json:"fp"`
|
||||
// Version is the version released at FP, which the want names beside it, once the put found it (design 47 H2 H-A-490
|
||||
// addendum).
|
||||
Version string `json:"version,omitempty"`
|
||||
// Hold keeps the want at FP until released.
|
||||
Hold bool `json:"hold,omitempty"`
|
||||
Why string `json:"why"`
|
||||
|
||||
@@ -5,9 +5,12 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -54,3 +57,14 @@ func TestABuildSeatsStepsAreReadInItsHoldersState(t *testing.T) {
|
||||
t.Errorf("a seat with no holder reads %v; want an error naming it", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-487: a build key's time is its entry's own write time, read with the key and no history.
|
||||
func TestABuildKeysTimeIsItsEntrysOwnWriteTime(t *testing.T) {
|
||||
at := time.Date(2026, 11, 22, 20, 45, 0, 0, time.UTC)
|
||||
raw, rev, got, err := entryOf(func(key string) (*stdio.StateEntry, error) {
|
||||
return &stdio.StateEntry{Key: key, Value: json.RawMessage(`{}`), Revision: 4, At: at}, nil
|
||||
})("k")
|
||||
if err != nil || string(raw) != "{}" || rev != 4 || !got.Equal(at) {
|
||||
t.Errorf("entry = %s rev %d at %v (%v); want the entry's own write time %v", raw, rev, got, err, at)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,7 +430,7 @@ func main() {
|
||||
return a, json.Unmarshal(raw, &a)
|
||||
}
|
||||
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr, reserve)
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds").Get), Asked: read(stdio.State("asked")),
|
||||
Newest: newestVia(stdio.Ask), Steps: r}))
|
||||
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
as := stdio.State(versions.ActedState)
|
||||
@@ -481,13 +481,14 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// entryOf reads a key with its revision and when it was written.
|
||||
func entryOf(s stdio.KeptState) func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
// entryOf reads a key with its revision and when it was written, the entry's own write time: no key's history is read
|
||||
// (design 47 H2 H-A-487).
|
||||
func entryOf(get func(key string) (*stdio.StateEntry, error)) func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
return func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
e, err := s.Get(key)
|
||||
e, err := get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, time.Time{}, err
|
||||
}
|
||||
return e.Value, e.Revision, written(s, key, e.Revision), nil
|
||||
return e.Value, e.Revision, e.At, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,6 +233,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
b.PackageWait = time.Duration(packageWait) * time.Minute
|
||||
// A version's digests are recorded under the builder's own bound, never the build's (H-A-492); it is record-bound-seconds.
|
||||
recordBound, err := numberFile(getenv("MESH_BUILDER_RECORD_BOUND_FILE"), int(build.DefaultRecordBound/time.Second))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.RecordBound = time.Duration(recordBound) * time.Second
|
||||
// Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194).
|
||||
b.PublishedVersions = publishedVersionsVia(calls.Call)
|
||||
waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
|
||||
@@ -254,11 +260,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// A package's number reserved is said as the seat mesh-builds' version-reserved, so the indexer re-pins at the reservation (§805).
|
||||
reserved.names = versions.NamesFrom(versions.EntriesOf(branches))
|
||||
reserved.r.Names = reserved.names // a Go package's module path names its tag's directory (§796)
|
||||
reserved.sayReserved = sayReservedOn(js, cred)
|
||||
b.Reserved = reserved
|
||||
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
|
||||
reserved.withdraw = b.Withdraw
|
||||
reserved.recordAgain = b.RecordAgain // a done build's digests its end did not record are recorded at the start (H-A-492)
|
||||
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
|
||||
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
@@ -298,8 +304,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
reserved.pause = publishSet.Pause // a version-released whose tag the forge did not write waits the pause (H-A-484 addendum)
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+16
-16
@@ -289,7 +289,8 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
var tagsWritten []string
|
||||
call := func(tool string, body any) (json.RawMessage, error) {
|
||||
switch tool {
|
||||
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796)
|
||||
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit, never read (H-A-494)
|
||||
tagsWritten = append(tagsWritten, "read "+body.(gitseat.TagsInput).Prefix)
|
||||
return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}})
|
||||
case "seat:" + gitseat.Seat + ".tag":
|
||||
tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit)
|
||||
@@ -360,15 +361,14 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects)
|
||||
}
|
||||
|
||||
// Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another
|
||||
// commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags <directory>/v<version> at
|
||||
// the build's commit, and the module's version-released clears the finding.
|
||||
rv.r.Names = func(context.Context, string, string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/m4", true, nil
|
||||
}
|
||||
// Design 47 H2 H-A-494: no git tag is read before a reserve nor written at a release: m4's 0.2.0, which the seat git would
|
||||
// answer tagged at another commit, is reserved, no finding raised, and its version-released writes no tag.
|
||||
// Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it
|
||||
// (sayReservedOn), and waited on by the seat and the event's name.
|
||||
rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"})
|
||||
rv.names = func(context.Context, string, string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/m4", true, nil
|
||||
}
|
||||
rv.sayReserved = sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"})
|
||||
said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -383,18 +383,18 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
if err := said.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib",
|
||||
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4",
|
||||
Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1})
|
||||
elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere)
|
||||
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised {
|
||||
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised)
|
||||
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.0" || v4.Tag != "" || raised {
|
||||
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.0 naming no tag, no tag read and no finding", v4, err, elsewhere, raised)
|
||||
}
|
||||
select {
|
||||
case m := <-reservedCh:
|
||||
ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data)
|
||||
if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" ||
|
||||
if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.0" ||
|
||||
ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule {
|
||||
t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source"))
|
||||
t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.0, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source"))
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved)
|
||||
@@ -420,11 +420,11 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
}
|
||||
return m
|
||||
}
|
||||
if !rv.heard(ctx, next("m4", "0.2.1")) {
|
||||
if !rv.heard(ctx, next("m4", "0.2.0")) {
|
||||
t.Fatal("the reservations did not take version-released")
|
||||
}
|
||||
if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) {
|
||||
t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten)
|
||||
if _, standing, _ := rv.r.Of(ctx, "m4", "0.2.0"); standing || len(tagsWritten) != 0 {
|
||||
t.Errorf("after version-released m4 0.2.0 standing %v, tags read or written %v: want it released and no tag read or written", standing, tagsWritten)
|
||||
}
|
||||
|
||||
// Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
@@ -57,11 +56,9 @@ type reservations struct {
|
||||
logf func(string, ...any)
|
||||
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
|
||||
withdraw func(module, fingerprint, why string)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
call toolCaller
|
||||
// pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands
|
||||
// it back at once.
|
||||
pause time.Duration
|
||||
// recordAgain records the digests of a done build of module at fp its end did not record, saying so (design 47 H2 H-A-492);
|
||||
// nil records none.
|
||||
recordAgain func(ctx context.Context, module, fp, when string) (string, error)
|
||||
// rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none.
|
||||
rebuild func(ctx context.Context, module, fp, why string) error
|
||||
// say says package-published on the seat mesh-builds; nil says none.
|
||||
@@ -120,7 +117,7 @@ func sayReservedOn(js jetstream.JetStream, cred process.Account) func(context.Co
|
||||
// newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's
|
||||
// verb versions through call (design 47 G §348), the only source numbering reads them from.
|
||||
func newReservations(store versions.ReserveStore, call toolCaller, bounds func() jetstream.KeyValue, found *findings, wake func(context.Context), logf func(string, ...any)) *reservations {
|
||||
rv := &reservations{call: call, bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf}
|
||||
rv := &reservations{bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf}
|
||||
rv.r = &versions.Reserver{Store: store, Filled: rv.filled.Load, ReleasedOf: func(_ context.Context, module string) ([]string, error) {
|
||||
raw, err := call("seat:"+meshcatalog.Seat+".versions", meshcatalog.VersionsInput{Module: module})
|
||||
if err != nil {
|
||||
@@ -137,8 +134,6 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func()
|
||||
return out, nil
|
||||
}}
|
||||
rv.r.Took, rv.r.Logf, rv.r.Reserved = rv.took, logf, rv.reservedSaid
|
||||
// A module's tags are read before it reserves, and a number tagged at another commit skipped (design 47 H2 §796).
|
||||
rv.r.Tags, rv.r.TaggedElsewhere = tagsVia(call), rv.taggedElsewhere
|
||||
return rv
|
||||
}
|
||||
|
||||
@@ -313,15 +308,9 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err)
|
||||
break
|
||||
}
|
||||
if refused, err := rv.release(ctx, ev); err != nil {
|
||||
if err := rv.release(ctx, ev); err != nil {
|
||||
rv.logf("%v; heard again", err)
|
||||
// A tag the seat git's holder did not write is handed back with the publish pause, so the deliveries the consumer
|
||||
// allows span the holder's outage (design 47 H2 H-A-484 addendum).
|
||||
nak := m.Nak
|
||||
if refused && rv.pause > 0 {
|
||||
nak = func() error { return m.NakWithDelay(rv.pause) }
|
||||
}
|
||||
if nakErr := nak(); nakErr != nil {
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf)
|
||||
@@ -336,16 +325,12 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// release takes a released version in as the catalog's version-released says it (design 47 G §369, §370): its tag written, its
|
||||
// reservation released, its bound and publish item ended and its findings cleared. refused says the tag was not written (the
|
||||
// seat git did not write it, or the reservation could not be read to tag it); the reservation then stands.
|
||||
func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) (refused bool, err error) {
|
||||
if err := rv.tagged(ctx, ev); err != nil {
|
||||
return true, err
|
||||
}
|
||||
// release takes a released version in as the catalog's version-released says it (design 47 G §369): its reservation released,
|
||||
// its bound and publish item ended and its findings cleared. No git tag is written (design 47 H2 H-A-494).
|
||||
func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error {
|
||||
gone, err := rv.r.Released(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return err
|
||||
}
|
||||
for _, k := range gone {
|
||||
rv.dropBound(ctx, k)
|
||||
@@ -355,20 +340,29 @@ func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionRelea
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
return false, nil
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) // one raised before H-A-494
|
||||
return nil
|
||||
}
|
||||
|
||||
// releasedAtStart takes in, at the builder's start, every built, unreleased reservation whose version the catalog's verb versions
|
||||
// holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one
|
||||
// said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended.
|
||||
// said while the builder was away, is taken all the same, its reservation released and its publish item ended; no tag is written
|
||||
// (design 47 H2 H-A-494).
|
||||
// A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released.
|
||||
// Before that, a standing reservation with no digests recorded whose build key is done has them recorded from the done key
|
||||
// (design 47 H2 H-A-492), so it is held against the released versions too.
|
||||
func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
all, err := rv.r.All(ctx)
|
||||
if err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
if rv.recordedAtStart(ctx, all) {
|
||||
if all, err = rv.r.All(ctx); err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
read := map[string][]string{}
|
||||
for _, v := range builtOf(all) {
|
||||
if versions.NotStanding(v, true) != "" {
|
||||
@@ -384,7 +378,7 @@ func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
if !slices.Contains(released, v.Version) {
|
||||
continue
|
||||
}
|
||||
if _, err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil {
|
||||
if err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil {
|
||||
rv.logf("%s %s is released in the catalog and not taken in at the start: %v", v.Module, v.Version, err)
|
||||
continue
|
||||
}
|
||||
@@ -392,6 +386,26 @@ func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// recordedAtStart records the digests of every standing reservation that has none recorded and whose build key is done (design 47
|
||||
// H2 H-A-492): a record its build's end lost. It answers whether one was recorded; one that cannot be is said.
|
||||
func (rv *reservations) recordedAtStart(ctx context.Context, all []versions.ReservedVersion) bool {
|
||||
if rv.recordAgain == nil {
|
||||
return false
|
||||
}
|
||||
recorded := false
|
||||
for _, v := range all {
|
||||
if !v.BuiltAt.IsZero() || v.Fingerprint == "" || versions.NotStanding(v, true) != "" {
|
||||
continue
|
||||
}
|
||||
version, err := rv.recordAgain(ctx, v.Module, v.Fingerprint, "at the builder's start")
|
||||
if err != nil {
|
||||
rv.logf("%v", err)
|
||||
}
|
||||
recorded = recorded || version != ""
|
||||
}
|
||||
return recorded
|
||||
}
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told
|
||||
// before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told
|
||||
// nothing.
|
||||
@@ -451,33 +465,6 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str
|
||||
return fmt.Sprintf("the reservation of %s %s is dropped (%s, %s)", module, series, by, why), nil
|
||||
}
|
||||
|
||||
// tagged writes the git tag of a released version (design 47 G §370): on the catalog's version-released, through the seat
|
||||
// git's verb tag, at the commit the reservation of that version was made from. A version with no reservation (a superseded
|
||||
// build's version is never released, and the key of a version released by another builder is gone) or one that names no
|
||||
// tag (a module with no Go package) is tagged nowhere. A tag the forge could not write is an error, the message heard again.
|
||||
func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error {
|
||||
res, found, err := rv.r.Of(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the reservation of %s %s cannot be read to tag it: %w", ev.Module, ev.Version, err)
|
||||
}
|
||||
if !found || res.Tag == "" || res.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
raw, err := rv.call("seat:"+git.Seat+".tag", git.TagInput{Repository: res.Repository, Commit: res.Commit, Tag: res.Tag})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the tag %s of %s is not written: %w", res.Tag, ev.Module, err)
|
||||
}
|
||||
var a git.TagAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return fmt.Errorf("the git seat answered what is no tag answer for %s: %w", res.Tag, err)
|
||||
}
|
||||
if a.Error != "" || !a.Performed {
|
||||
return fmt.Errorf("the tag %s of %s is not written: %s (%s)", res.Tag, ev.Module, a.Error, a.ErrorKind)
|
||||
}
|
||||
rv.logf("%s is tagged %s at %s", ev.Module, res.Tag, res.Commit)
|
||||
return nil
|
||||
}
|
||||
|
||||
// noMark says why a published marked nothing, from the reservations the published read (design 47 H H-A-280: no second read):
|
||||
// what each holds of digests, and how many stand at that version by the index by version (its one key, no reservation read).
|
||||
func (rv *reservations) noMark(ctx context.Context, o versions.Outcome, read []versions.ReadReservation) string {
|
||||
@@ -549,19 +536,6 @@ func (rv *reservations) heldElsewhere(ctx context.Context, module, why string) {
|
||||
needs: "builder_reserved shows the reservation and its withdrawn numbers; the registry seat's versions shows what it holds."})
|
||||
}
|
||||
|
||||
// taggedElsewhere raises version-tagged-elsewhere about module (design 47 H2 §796): numbers whose git tag stands at another
|
||||
// commit were skipped at reserving; a tag is never moved. Cleared at the module's next version-released.
|
||||
func (rv *reservations) taggedElsewhere(ctx context.Context, module string, skipped []string) {
|
||||
why := "the tags of " + strings.Join(skipped, ", ") + " stand at another commit, so the reserve skipped them"
|
||||
rv.logf("%s: %s", module, why)
|
||||
rv.found.raise(ctx, finding{kind: versions.VersionTaggedElsewhere, scope: meshconditionsclient.ScopeModule, id: module,
|
||||
summary: module + ": " + why,
|
||||
headline: "A number whose tag stands at another commit is not used",
|
||||
explanation: "The repository holds a tag of " + module + " at a commit other than the build's: " + why + ". A tag is never moved.",
|
||||
resolved: "The catalog says version-released for " + module + ".",
|
||||
needs: "builder_reserved shows the reservation; the seat git's tags shows each tag with its commit."})
|
||||
}
|
||||
|
||||
// movedElsewhere takes a done build whose number a registry holds at another digest off that number (design 47 H H-A-275,
|
||||
// H-A-279): every artifact carries its version from its build, so no digest survives a move. The number is withdrawn as a gap
|
||||
// (H-A-226), the finding raised naming both digests, and the build of the same fingerprint asked again (the build-again path of a
|
||||
|
||||
@@ -156,123 +156,117 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom: design 47 G §370 — the git tag go/vMAJOR.MINOR.PATCH is written through
|
||||
// the seat git's tag on the catalog's version-released, at the commit the version's reservation was made from; a module with no
|
||||
// Go package, and a version never released (a superseded build's), are tagged nowhere; a tag the forge did not write leaves
|
||||
// version-released to be heard again with the reservation standing.
|
||||
func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
// TestAReleasedVersionWritesNoTag: design 47 H2 H-A-494 — the builder writes no git tag at a release, and numbers a series from the
|
||||
// catalog's versions and the registries' alone: the catalog's version-released releases the reservation with no call of the seat
|
||||
// git, a reservation written before (one naming a tag) included; the start's take-in of a version the catalog holds released
|
||||
// (H-A-484 c) the same; and a reserve reads no git tag, a tag standing at another commit no reason to skip a number.
|
||||
func TestAReleasedVersionWritesNoTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var tags []git.TagInput
|
||||
refuse := ""
|
||||
var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796)
|
||||
var gitCalls []string // every call of the seat git: none is made
|
||||
var catalog []meshcatalog.VersionEntry // the released versions the catalog's verb versions answers
|
||||
call := func(addr string, body any) (json.RawMessage, error) {
|
||||
switch addr {
|
||||
case "seat:git.tags":
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true})
|
||||
case "seat:git.tag":
|
||||
default:
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog})
|
||||
if strings.HasPrefix(addr, "seat:git.") {
|
||||
gitCalls = append(gitCalls, addr)
|
||||
if addr == "seat:git.tags" { // a tag standing at another commit, as the forge may still hold one
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}}})
|
||||
}
|
||||
return json.Marshal(git.TagAnswer{Performed: true})
|
||||
}
|
||||
tags = append(tags, body.(git.TagInput))
|
||||
return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse})
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog})
|
||||
}
|
||||
fw := &memFindings{vals: map[string][]byte{}}
|
||||
found := &findings{module: builds.BuilderModule, w: fw, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
store := &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
rv := newReservations(store, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
// Design 47 H2 §796: the tag names the Go module's directory, read from its module path: mesh-sdk's `go` here.
|
||||
rv.r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/go", artifact == "lib" || artifact == "pkg", nil
|
||||
rv.names = func(_ context.Context, _, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/go", artifact == "lib", nil
|
||||
}
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true, GoArtifact: "lib"}
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rv.Reserve(ctx, "prog", "2.0", []string{"prog:bin"}, versions.Source{Repository: "o/p", Commit: "beef"}); err != nil {
|
||||
t.Fatal(err)
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", Fingerprint: "fpA"}
|
||||
// oldRecord rewrites a reservation as one written before H-A-494, naming the tag its release was to be written as.
|
||||
oldRecord := func(module, series, tag string) {
|
||||
key := versions.ReservedKey(module, series)
|
||||
var v versions.ReservedVersion
|
||||
if err := json.Unmarshal(store.vals[key], &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.Tag = tag
|
||||
store.vals[key], _ = json.Marshal(v)
|
||||
}
|
||||
release := func(module, version string) *wakeMsg {
|
||||
m := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"` + module + `","version":"` + version + `","source":1,"artifacts":[]}`)}
|
||||
rv.heard(ctx, m)
|
||||
return m
|
||||
}
|
||||
if m := release("prog", "2.0.0"); !m.acked || len(tags) != 0 {
|
||||
t.Fatalf("a module with no Go package: acked %v, tags %v; want none written", m.acked, tags)
|
||||
// The release path: a Go package's version-released releases it, acknowledged, with no tag written.
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil || v.Tag != "" {
|
||||
t.Fatalf("reserved %+v (%v), want a reservation naming no tag", v, err)
|
||||
}
|
||||
refuse = "forge down"
|
||||
if m := release("pkg", "0.1.0"); !m.naked || m.acked {
|
||||
t.Fatalf("a tag the forge did not write: naked %v acked %v; want the release heard again", m.naked, m.acked)
|
||||
oldRecord("pkg", "0.1", "go/v0.1.0")
|
||||
if m := release("pkg", "0.1.0"); !m.acked || m.naked {
|
||||
t.Fatalf("the release: acked %v naked %v; want it acknowledged", m.acked, m.naked)
|
||||
}
|
||||
// Design 47 H2 H-A-484 addendum: with a publish pause set, a refused tag hands the version-released back after the pause, so
|
||||
// the deliveries its consumer allows span the holder's outage.
|
||||
rv.pause = 30 * time.Second
|
||||
if m := release("pkg", "0.1.0"); m.delayed != 30*time.Second || m.naked || m.acked {
|
||||
t.Fatalf("a tag the forge did not write, with the pause: delayed %v naked %v acked %v; want handed back after 30s", m.delayed, m.naked, m.acked)
|
||||
if len(gitCalls) != 0 {
|
||||
t.Errorf("the release called the seat git: %v; want no tag written", gitCalls)
|
||||
gitCalls = nil
|
||||
}
|
||||
rv.pause = 0
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("the reservation went with an unwritten tag: %v", all)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 {
|
||||
t.Fatalf("the released version still stands: %v", all)
|
||||
}
|
||||
// Design 47 H2 H-A-484 addendum: at the builder's start, a built reservation the catalog holds released is tagged and released
|
||||
// as its version-released would (a version-released the bus gave up on), and one the catalog does not hold, or not built,
|
||||
// stands.
|
||||
refuse = ""
|
||||
tags = nil
|
||||
// The start check (H-A-484 c): a built reservation the catalog holds released is released at the start, no tag written; one
|
||||
// not built, or not released, stands.
|
||||
// Design 47 H2 H-A-492: at the start, a standing reservation with no digests recorded whose build key is done has them
|
||||
// recorded (the builder's RecordAgain), and is then held against the released versions as a built one is.
|
||||
if _, err := rv.Reserve(ctx, "rec", "0.1", []string{"rec:bin"}, versions.Source{Repository: "o/q", Commit: "abc", Fingerprint: "fpR"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var recordedAt []string
|
||||
rv.recordAgain = func(ctx context.Context, module, fp, when string) (string, error) {
|
||||
recordedAt = append(recordedAt, module+" "+fp+" "+when)
|
||||
_, err := rv.r.Built(ctx, module, "0.1.0", "rec:bin", "sha256:bb", nil) // its done key's digest
|
||||
return "0.1.0", err
|
||||
}
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "rec", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if _, standing, _ := rv.r.Of(ctx, "rec", "0.1.0"); standing || !slices.Equal(recordedAt, []string{"rec fpR at the builder's start"}) {
|
||||
t.Fatalf("at the start, rec 0.1.0 unrecorded and released: standing %v, recorded %q; want it recorded once at the start and released", standing, recordedAt)
|
||||
}
|
||||
catalog, rv.recordAgain = nil, nil
|
||||
var ended []string
|
||||
rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) }
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", Fingerprint: "fpB"}); err != nil {
|
||||
t.Fatal(err) // 0.1.0 again: the stub catalog holds none released
|
||||
}
|
||||
oldRecord("pkg", "0.1", "go/v0.1.0")
|
||||
rv.releasedAtStart(ctx) // built nothing yet: nothing taken
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 {
|
||||
t.Fatalf("at the start with no build done: reservations %v, tags %+v; want the reservation standing, no tag", all, tags)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("at the start with no build done: reservations %v; want the reservation standing", all)
|
||||
}
|
||||
if _, err := rv.r.Built(ctx, "pkg", "0.1.0", "pkg:lib", "sha256:aa", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.releasedAtStart(ctx) // the catalog holds none released: it stands
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 {
|
||||
t.Fatalf("at the start with nothing released: reservations %v, tags %+v; want the reservation standing", all, tags)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("at the start with nothing released: reservations %v; want the reservation standing", all)
|
||||
}
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "pkg", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) || !slices.Equal(ended, []string{"pkg 0.1.0"}) {
|
||||
t.Fatalf("at the start with 0.1.0 released: reservations %v, tags %+v, items ended %v; want it tagged, released and its item ended", all, tags, ended)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 || !slices.Equal(ended, []string{"pkg 0.1.0"}) {
|
||||
t.Fatalf("at the start with 0.1.0 released: reservations %v, items ended %v; want it released and its item ended", all, ended)
|
||||
}
|
||||
if len(gitCalls) != 0 {
|
||||
t.Errorf("the start check called the seat git: %v; want no tag written", gitCalls)
|
||||
gitCalls = nil
|
||||
}
|
||||
catalog, rv.released = nil, nil
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // the version is reserved again for the rows below
|
||||
t.Fatal(err)
|
||||
}
|
||||
tags = nil
|
||||
if m := release("pkg", "0.1.0"); !m.acked || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) {
|
||||
t.Fatalf("the release: acked %v, tags %+v; want go/v0.1.0 at c0ffee of o/r", m.acked, tags)
|
||||
}
|
||||
tags = nil
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // build A: 0.1.0 (the stub catalog holds none released)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true, GoArtifact: "lib"}); err != nil || v.Version != "0.1.1" { // build B supersedes A
|
||||
t.Fatalf("second build reserved %q (%v)", v.Version, err)
|
||||
}
|
||||
release("pkg", "0.1.0") // A's version is never released in fact; were it said, it is no reservation of the key
|
||||
if len(tags) != 0 {
|
||||
t.Fatalf("a superseded build's version was tagged: %+v", tags)
|
||||
}
|
||||
if release("pkg", "0.1.1"); len(tags) != 1 || tags[0].Commit != "b0b" {
|
||||
t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags)
|
||||
}
|
||||
// Design 47 H2 §796: a tag standing at another commit makes the next number and the finding version-tagged-elsewhere, which
|
||||
// clears at the module's next version-released.
|
||||
standing = []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}, {Tag: "go/v0.2.1", Commit: "elsewhere"}}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.2" || v.Tag != "go/v0.2.2" {
|
||||
t.Fatalf("reserved %+v (%v) with 0.2.0 and 0.2.1 tagged elsewhere, want 0.2.2 tagged go/v0.2.2", v, err)
|
||||
// Numbering reads no tag: the seat git would answer go/v0.2.0 at another commit, and 0.2.0 is reserved all the same.
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.0" || v.Tag != "" || len(gitCalls) != 0 {
|
||||
t.Errorf("reserved %+v (%v), git calls %v; want 0.2.0 naming no tag, no git tag read", v, err, gitCalls)
|
||||
}
|
||||
key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "pkg", versions.VersionTaggedElsewhere)
|
||||
if got, err := decodeFinding(fw.vals[key]); err != nil || !strings.Contains(got.Summary, "0.2.0") || !strings.Contains(got.Summary, "0.2.1") {
|
||||
t.Fatalf("the finding %s = %+v (%v), want it naming 0.2.0 and 0.2.1", key, got, err)
|
||||
if fw.vals[key] != nil {
|
||||
t.Errorf("a reserve raised %s", key)
|
||||
}
|
||||
if release("pkg", "0.2.2"); fw.vals[key] != nil {
|
||||
t.Errorf("version-released of pkg left %s standing", key)
|
||||
}
|
||||
standing = nil
|
||||
// Design 47 H2 §805, H-A-420 (2): a package's number newly reserved is said as version-reserved, its path the package's name
|
||||
// and its revision the indexer's `<branch>.<n>`; a reservation kept as it stood, an artifact that is no package and a
|
||||
// reservation naming no catalog revision say none.
|
||||
@@ -281,7 +275,7 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
return "git.example/o/r/" + artifact, module == "lib2" && artifact == "pkg", nil
|
||||
}
|
||||
rv.sayReserved = func(_ context.Context, ev meshbuilds.VersionReservedEvent) error { said = append(said, ev); return nil }
|
||||
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", GoPackage: true, GoArtifact: "pkg", Branch: "main", CatalogRevision: 158}
|
||||
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", Branch: "main", CatalogRevision: 158}
|
||||
for range 2 {
|
||||
if _, err := rv.Reserve(ctx, "lib2", "0.3", []string{"lib2:pkg", "lib2:bin"}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -19,6 +19,8 @@ import (
|
||||
// `module` alone the module's keys newest first, at most MostBuilds, each with its fingerprint whole, its state, revision and
|
||||
// since when, its step and node, its error whole, event-builds and the open ask (§568), and `newest` naming each watched branch
|
||||
// whose newest catalog graph entry of the module names that fingerprint, read from the catalog half by the tools bundle.
|
||||
// **Each key's time is its entry's own write time** (design 47 H2 H-A-487): the history of a key is read only for the one key a
|
||||
// fingerprint names, when its entry does not say when it was written, and a listing cut at MostBuilds says so in its note.
|
||||
|
||||
// BuildTool is the builder's read tool of a module's build keys.
|
||||
const BuildTool = "builder_build"
|
||||
@@ -56,14 +58,16 @@ type BuildAnswer struct {
|
||||
Builds []BuildRow `json:"builds"`
|
||||
// Newest is the fingerprint of the module's newest catalog graph entry on each watched branch.
|
||||
Newest map[string]string `json:"newest-entries,omitempty"`
|
||||
// Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint.
|
||||
// Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint, and that the listing
|
||||
// was cut at MostBuilds (H-A-487).
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// BuildReader reads what builder_build answers.
|
||||
type BuildReader struct {
|
||||
Builds StateRead
|
||||
// Entry is a key's value with its revision and when it was written; nil leaves both out.
|
||||
// Entry is a key's value with its revision and when it was written, the entry's own write time (H-A-487); nil leaves both
|
||||
// out. Builds.Written, the key's history, is read only for the key a fingerprint names when Entry says no time.
|
||||
Entry func(key string) (json.RawMessage, uint64, time.Time, error)
|
||||
// Asked is the builder's state asked, which holds the asks the operator was asked.
|
||||
Asked StateRead
|
||||
@@ -96,6 +100,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
if raw == nil {
|
||||
continue
|
||||
}
|
||||
if at.IsZero() && fingerprint != "" && r.Builds.Written != nil {
|
||||
at = r.Builds.Written(key) // one key's history, never every key's (H-A-487)
|
||||
}
|
||||
v, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%s holds what is not a build: %w", key, err)
|
||||
@@ -118,7 +125,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
}
|
||||
return rows[i].Key > rows[j].Key
|
||||
})
|
||||
var cut string
|
||||
if len(rows) > MostBuilds {
|
||||
cut = fmt.Sprintf("%d of %s's %d build keys are answered, the newest by write time; name a fingerprint for another", MostBuilds, module, len(rows))
|
||||
rows = rows[:MostBuilds]
|
||||
}
|
||||
asks := r.openAsks()
|
||||
@@ -149,6 +158,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
out.Note = "no build key of " + module + " is the newest entry's: " + strings.Join(says, "; ")
|
||||
}
|
||||
}
|
||||
if cut != "" {
|
||||
out.Note = strings.TrimPrefix(out.Note+"; "+cut, "; ")
|
||||
}
|
||||
out.Builds = append(out.Builds, rows...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -66,6 +67,31 @@ func TestBuilderBuildAnswersAModulesKeysWhole(t *testing.T) {
|
||||
if n, err := r.Build("m", ""); err != nil || !strings.Contains(n.Note, "no build key of m is the newest entry's") || !strings.Contains(n.Note, "main names src2:"+strings.Repeat("c", 64)) {
|
||||
t.Errorf("a module none of whose keys the newest entry names: %+v, %v", n, err)
|
||||
}
|
||||
|
||||
// H-A-487: a module of 100 keys is read by each entry's own write time, no key's history read, and the listing is cut at the
|
||||
// 20 newest, saying so; a fingerprint's key with no time in its entry has its one history read.
|
||||
many, at := map[string][]byte{}, map[string]time.Time{}
|
||||
base := time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC)
|
||||
for i := range 100 {
|
||||
k, _ := builds.BuilderBuildKey("big", fmt.Sprintf("src2:%064x", i))
|
||||
many[k], at[k] = done, base.Add(time.Duration(i)*time.Minute)
|
||||
}
|
||||
histories := 0
|
||||
st := stateOf(many)
|
||||
st.Written = func(string) time.Time { histories++; return base }
|
||||
big := &BuildReader{Builds: st, Entry: func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, at[k], nil }}
|
||||
a, err = big.Build("big", "")
|
||||
if err != nil || len(a.Builds) != MostBuilds || !a.Builds[0].Since.Equal(base.Add(99*time.Minute)) || histories != 0 ||
|
||||
!strings.Contains(a.Note, "20 of big's 100 build keys are answered") {
|
||||
t.Fatalf("100 keys: %d rows, newest %v, %d histories read, note %q; want the 20 newest, no history, the cut said", len(a.Builds), a.Builds[0].Since, histories, a.Note)
|
||||
}
|
||||
big.Entry = func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, time.Time{}, nil }
|
||||
if a, err := big.Build("big", ""); err != nil || len(a.Builds) != MostBuilds || histories != 0 {
|
||||
t.Fatalf("100 keys with no time in their entries: %d rows (%v), %d histories read; want no history read for a listing", len(a.Builds), err, histories)
|
||||
}
|
||||
if one, err := big.Build("big", fmt.Sprintf("src2:%064x", 7)); err != nil || len(one.Builds) != 1 || !one.Builds[0].Since.Equal(base) || histories != 1 {
|
||||
t.Errorf("one fingerprint with no time in its entry: %+v (%v), %d histories; want its one history read", one, err, histories)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §589: builder_stuck answers each key's fingerprint whole, as the build-input fingerprint the catalog graph entry carries it.
|
||||
|
||||
@@ -170,6 +170,7 @@ func Unreferenced(b builds.BuilderBuild) bool {
|
||||
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
|
||||
// which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
|
||||
// another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder);
|
||||
// - a failed key withdrawn (withdrawnPrefix) is final for its fingerprint, taken again only by a person's build again (H-A-489);
|
||||
// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is
|
||||
// published (ClaimOptions.Published, ADR 0500 clause 6), and not before;
|
||||
// - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its
|
||||
@@ -229,6 +230,8 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)):
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, withdrawnPrefix) && !opt.ByHand:
|
||||
return nil, nil // withdrawn is final for its fingerprint: no look builds it again (H-A-489)
|
||||
case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver && !opt.ByEvent:
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
|
||||
|
||||
@@ -106,11 +106,19 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
|
||||
return nil
|
||||
}
|
||||
raw, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
if err != nil {
|
||||
b.logf("%s: the build key cannot be read, so it is taken as not done: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
cur, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil || cur.State != builds.BuildDone {
|
||||
if err != nil {
|
||||
b.logf("%s: the build key holds what is no build, so it is taken as not done: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if cur.State != builds.BuildDone {
|
||||
return nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
@@ -122,6 +130,57 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
|
||||
return out
|
||||
}
|
||||
|
||||
// DefaultRecordBound is the bound of the record of a reservation's digests, its setting record-bound-seconds' default (design 47
|
||||
// H2 H-A-492).
|
||||
const DefaultRecordBound = time.Minute
|
||||
|
||||
// recordContext is the context a reservation's digests are recorded under: ctx's values with no cancel of its own, bounded by
|
||||
// RecordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record.
|
||||
func (b *Builder) recordContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
bound := b.RecordBound
|
||||
if bound <= 0 {
|
||||
bound = DefaultRecordBound
|
||||
}
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), bound)
|
||||
}
|
||||
|
||||
// RecordAgain records the digests of fp's done build under its standing reservation when its end did not record them (design 47
|
||||
// H2 H-A-492): at the builder's look of the key and at its start (when says which), said each time. It answers the version
|
||||
// recorded, "" when nothing was to be recorded: the key is not done, no reservation stands for fp, or it is recorded already.
|
||||
func (b *Builder) RecordAgain(ctx context.Context, module, fp, when string) (string, error) {
|
||||
b.init()
|
||||
if b.Reserved == nil {
|
||||
return "", nil
|
||||
}
|
||||
digests := b.doneDigests(ctx, module, fp)
|
||||
if digests == nil {
|
||||
return "", nil
|
||||
}
|
||||
res, ok, err := b.Reserved.ReservedFor(ctx, module, fp)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s at %s: whether its done build's digests are recorded cannot be read %s: %w", module, fp, when, err)
|
||||
}
|
||||
if !ok || !res.BuiltAt.IsZero() || len(res.Artifacts) == 0 || versions.NotStanding(res, true) != "" {
|
||||
return "", nil
|
||||
}
|
||||
refs := map[string]string{}
|
||||
for ref := range res.Artifacts {
|
||||
_, artifact, _ := strings.Cut(ref, ":")
|
||||
d, have := digests[artifact]
|
||||
if !have {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded, and the done key holds no digest of %s to record %s", module, fp, res.Version, ref, when)
|
||||
}
|
||||
refs[ref] = d
|
||||
}
|
||||
rec, stop := b.recordContext(ctx)
|
||||
defer stop()
|
||||
if err := b.Reserved.Built(rec, module, res.Version, refs, nil); err != nil {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded %s either: %w", module, fp, res.Version, when, err)
|
||||
}
|
||||
b.logf("%s at %s: the digests of %s were not recorded at its build's end: recorded %s from the done key", module, fp, res.Version, when)
|
||||
return res.Version, nil
|
||||
}
|
||||
|
||||
// PackageNamed looks at a module again when the indexer's packages state names a name for one of its packages
|
||||
// (versions.NamedModule): a done trunk build that left the package unnumbered, its name not known at its claim, is
|
||||
// numbered then with the done build's digests, and version.numbered says them for its build module's late publish.
|
||||
@@ -199,8 +258,8 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
|
||||
}
|
||||
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
|
||||
for _, t := range plan.Tasks {
|
||||
if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage {
|
||||
src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796)
|
||||
if t.Kind == catalogue.ArtifactUpstream {
|
||||
src.Upstream = append(src.Upstream, module+":"+t.Artifact) // copied by its pin, never awaited (H-A-486)
|
||||
}
|
||||
}
|
||||
res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs, src)
|
||||
|
||||
@@ -32,6 +32,8 @@ type fakeReservations struct {
|
||||
next string // the version Reserve answers when set, else the series' .5
|
||||
fail error // the reservation cannot be made (the catalog's versions cannot be read)
|
||||
tookBy map[string]versions.ReservedVersion // by fingerprint, the reservation that took its series over (H-A-244)
|
||||
held map[string]versions.ReservedVersion // by fingerprint, the reservation ReservedFor answers as it stands (H-A-492)
|
||||
lost error // the record of the digests fails, as a store refusing it would
|
||||
}
|
||||
|
||||
func (f *fakeReservations) TakenOver(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
@@ -46,6 +48,9 @@ func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, e
|
||||
func (f *fakeReservations) Filled() bool { return f.filled }
|
||||
|
||||
func (f *fakeReservations) ReservedFor(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
if v, ok := f.held[fp]; ok {
|
||||
return v, true, nil
|
||||
}
|
||||
for _, s := range f.sources {
|
||||
if s.Fingerprint == fp {
|
||||
return versions.ReservedVersion{Version: "held"}, true, nil
|
||||
@@ -65,7 +70,13 @@ func (f *fakeReservations) Reserve(_ context.Context, module, series string, art
|
||||
return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
func (f *fakeReservations) Built(ctx context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err // the state's write runs under ctx: a cancelled one records nothing
|
||||
}
|
||||
if f.lost != nil {
|
||||
return f.lost
|
||||
}
|
||||
f.deps = deps
|
||||
f.built = map[string]string{version: ""}
|
||||
for ref, d := range digests {
|
||||
@@ -118,6 +129,13 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under the reserved version", res.built)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved for the record and named as upstream, so its publish is never awaited.
|
||||
up := &fakeReservations{filled: true}
|
||||
plan := Plan{Series: "0.1", Tasks: []Task{{Artifact: "server", Kind: catalogue.ArtifactUpstream}, {Artifact: "tools", Kind: catalogue.ArtifactBundle}}}
|
||||
if _, ok := (&Builder{Reserved: up}).reserve(context.Background(), "nats", meshcatalog.ModulesByModuleValue{Fp2: "f"}, plan, Origin{}, false); !ok ||
|
||||
!slices.Equal(up.reserved[0], []string{"nats:server", "nats:tools"}) || !slices.Equal(up.sources[0].Upstream, []string{"nats:server"}) {
|
||||
t.Errorf("an upstream artifact reserved %v from %+v, want both refs reserved and nats:server named upstream", up.reserved, up.sources)
|
||||
}
|
||||
r.consider(t, "modules/m")
|
||||
if len(res.reserved) != 1 {
|
||||
t.Errorf("a done key reserved again: %v", res.reserved)
|
||||
@@ -146,6 +164,97 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound: design 47 H2 H-A-492 — the record of a reservation's digests after a
|
||||
// build ends runs under the builder's own bound, so a build context cancelled at its end still records; a done build whose
|
||||
// digests are not recorded is recorded at the builder's next look, said.
|
||||
func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) {
|
||||
manifest := catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
t.Run("a build's context cancelled at its end still records", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
r.b.Ended = func(context.Context, string) { cancel() } // a takeover or the builder's end, once the key is written
|
||||
r.b.Module(ctx, "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
||||
t.Fatalf("build = %+v, want done", v)
|
||||
}
|
||||
if res.built["m:tool"] != digest {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under 0.1.5 though the build's context ended", res.built)
|
||||
}
|
||||
})
|
||||
t.Run("the record's bound is the setting record-bound-seconds, never the build's", func(t *testing.T) {
|
||||
build, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, c := range []struct {
|
||||
bound, want time.Duration
|
||||
}{{0, DefaultRecordBound}, {5 * time.Second, 5 * time.Second}} {
|
||||
rec, stop := (&Builder{RecordBound: c.bound}).recordContext(build)
|
||||
deadline, has := rec.Deadline()
|
||||
stop()
|
||||
if left := time.Until(deadline); !has || left > c.want || left < c.want-time.Second {
|
||||
t.Errorf("RecordBound %v: the record's context has deadline %v (%v left, %v), want %v left of its own", c.bound, deadline, left, has, c.want)
|
||||
}
|
||||
}
|
||||
rec, stop := (&Builder{}).recordContext(build)
|
||||
defer stop()
|
||||
if rec.Err() != nil {
|
||||
t.Errorf("the record's context ended with the build's: %v", rec.Err())
|
||||
}
|
||||
})
|
||||
t.Run("a build key that cannot be read is said, not taken silently", func(t *testing.T) {
|
||||
var journal []string
|
||||
b := &Builder{Claimer: Claimer{Store: unreadable{newMem()}}, Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }}
|
||||
if d := b.doneDigests(context.Background(), "m", fp1); d != nil {
|
||||
t.Errorf("an unreadable key answered digests %v", d)
|
||||
}
|
||||
if !slices.ContainsFunc(journal, func(l string) bool {
|
||||
return strings.Contains(l, "cannot be read") && strings.Contains(l, "bus timeout")
|
||||
}) {
|
||||
t.Errorf("journal %q, want a line saying the build key cannot be read, with why", journal)
|
||||
}
|
||||
})
|
||||
t.Run("a done key whose digests are not recorded is recorded at the next look", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true, lost: errors.New("context canceled")}
|
||||
r.b.Reserved = res
|
||||
var journal []string
|
||||
r.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
|
||||
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone || res.built != nil {
|
||||
t.Fatalf("build %s recorded %v, want done and nothing recorded", v.State, res.built)
|
||||
}
|
||||
res.lost = nil
|
||||
res.held = map[string]versions.ReservedVersion{fp1: {Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": ""}}}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v at the next look, want the done key's digest of m:tool under 0.1.5", res.built)
|
||||
}
|
||||
if !slices.ContainsFunc(journal, func(l string) bool {
|
||||
return strings.Contains(l, "0.1.5 were not recorded") && strings.Contains(l, "recorded at a look")
|
||||
}) {
|
||||
t.Errorf("journal %q, want a line saying 0.1.5's digests were recorded at a look", journal)
|
||||
}
|
||||
// Recorded once: a reservation whose build ended recorded is left.
|
||||
res.built = nil
|
||||
res.held[fp1] = versions.ReservedVersion{Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": digest}, BuiltAt: time.Unix(1, 0)}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built != nil {
|
||||
t.Errorf("a recorded reservation was recorded again: %v", res.built)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// unreadable is a builds bucket whose reads fail, as one on a bus that does not answer.
|
||||
type unreadable struct{ *memStore }
|
||||
|
||||
func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) {
|
||||
return nil, 0, false, errors.New("bus timeout")
|
||||
}
|
||||
|
||||
// TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through
|
||||
// reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why.
|
||||
func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) {
|
||||
@@ -236,7 +345,7 @@ func TestAReservedBuildKeepsWhatItsStepResolvedAndWasBuiltWith(t *testing.T) {
|
||||
|
||||
// TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion: design 47 G §370 — a module whose one artifact is
|
||||
// a Go package (mesh-wireguard's go-overlay) is planned on the Go build seat, its ask carries the reserved version and names
|
||||
// the seat go-package-registry the package is published and tagged through, and the build is done; a build reserving none
|
||||
// the seat go-package-registry the package is published through, and the build is done; a build reserving none
|
||||
// names no seat.
|
||||
func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *testing.T) {
|
||||
goPackage := catalogue.Artifact{Name: "go-overlay", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}}
|
||||
@@ -265,10 +374,6 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t
|
||||
len(asked[0].Build.Artifacts) != 1 || asked[0].Build.Artifacts[0] != (buildstep.AskArtifact{Artifact: "go-overlay", Seat: c.seat}) {
|
||||
t.Fatalf("asks %+v, want one Go ask with version %q naming go-overlay on seat %q", asked, c.version, c.seat)
|
||||
}
|
||||
// Design 47 H2 §796: the reservation names the Go package's artifact, whose module path names its tag's directory.
|
||||
if c.version != "" && (len(res.sources) != 1 || !res.sources[0].GoPackage || res.sources[0].GoArtifact != "go-overlay") {
|
||||
t.Errorf("reserved from %+v, want one source naming the Go package go-overlay", res.sources)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +107,9 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun,
|
||||
}
|
||||
}
|
||||
// Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture
|
||||
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded.
|
||||
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded. Once the picture
|
||||
// heard a newer entry of the module on the first branch, H-A-489 overrides H-A-205: the older fingerprint is refused
|
||||
// superseded (buildEntry), never built or reserved.
|
||||
b.mu.Lock()
|
||||
for _, e := range run.Entries {
|
||||
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
|
||||
|
||||
@@ -127,6 +127,9 @@ type Builder struct {
|
||||
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished)
|
||||
// gives a pin (H-A-192).
|
||||
PackageWait time.Duration
|
||||
// RecordBound bounds the record of a reservation's digests after a build ends, its setting record-bound-seconds (design 47
|
||||
// H2 H-A-492): the builder's own bound, never the build's context; DefaultRecordBound when 0.
|
||||
RecordBound time.Duration
|
||||
// PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first
|
||||
// (H-A-194): read once before a build waits for that package. Nil reads none.
|
||||
PublishedVersions func(ctx context.Context, path string) ([]string, error)
|
||||
@@ -177,6 +180,8 @@ type Builder struct {
|
||||
// publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished):
|
||||
// a package with no pin waits for one (H-A-192).
|
||||
publishedPins map[string]map[string]string
|
||||
// supersededSaid are the build keys refused as superseded (H-A-489), so each is said once.
|
||||
supersededSaid sync.Map
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -310,11 +315,49 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
|
||||
if superseded != nil {
|
||||
// A newer fingerprint of the module: the build of the older one in flight is taken out, and its asks are withdrawn
|
||||
// from the build seats (ADR 0664 §4).
|
||||
superseded(buildrun.Withdrawn{Why: "the build of " + key + " at " + old.Fp2 + " was superseded by the fingerprint " + m.Fp2})
|
||||
superseded(buildrun.Withdrawn{Why: supersededWhy(key, old.Fp2, m.Fp2)})
|
||||
}
|
||||
b.consider(ctx, key, m)
|
||||
}
|
||||
|
||||
// supersededWhy is the words of a build withdrawn because a newer fingerprint of its module superseded it.
|
||||
func supersededWhy(module, fp, newer string) string {
|
||||
return "the build of " + module + " at " + fp + " was superseded by the fingerprint " + newer
|
||||
}
|
||||
|
||||
// withdrawnPrefix opens the error of a key whose build was withdrawn (H-A-225, ADR 0664 §4). It is final for its fingerprint
|
||||
// (H-A-489): no look builds it again; a person's build again does.
|
||||
const withdrawnPrefix = "withdrawn: "
|
||||
|
||||
// supersededBy is the fingerprint that supersedes fp, a fingerprint of module whose build key is key, when fp is of its
|
||||
// registration's first watched branch and is no longer the newest entry heard on that branch (H-A-489); "" otherwise. A key no
|
||||
// entry named is of the first branch, as originOf has it.
|
||||
func (b *Builder) supersededBy(module, key, fp string) string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
o, ok := b.origin[key]
|
||||
if !ok {
|
||||
o = Origin{First: true}
|
||||
}
|
||||
if !o.First {
|
||||
return ""
|
||||
}
|
||||
newest, heard := b.heard[o.Branch+"\x00"+module]
|
||||
if !heard || newest.Fp2 == "" || newest.Fp2 == fp {
|
||||
return ""
|
||||
}
|
||||
return newest.Fp2
|
||||
}
|
||||
|
||||
// saySuperseded says, once per key, that the build of fp is refused as superseded by newer (H-A-489), and gives the words.
|
||||
func (b *Builder) saySuperseded(module, key, fp, newer string) string {
|
||||
why := "is not built: superseded by " + newer + ", the newest entry heard on its first branch"
|
||||
if _, said := b.supersededSaid.LoadOrStore(key, true); !said {
|
||||
b.logf("%s at %s: %s", module, fp, why)
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
|
||||
// directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is
|
||||
// built against, its packages and needs, and the named paths of its artifacts when every artifact names them.
|
||||
@@ -637,6 +680,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
whyEnded = fmt.Sprintf(format, args...)
|
||||
b.logf("%s at %s: %s", module, m.Fp2, whyEnded)
|
||||
}
|
||||
// A fingerprint no longer the newest entry heard on its first branch is neither reserved for nor built (H-A-489): two
|
||||
// fingerprints of one module never take the series from each other.
|
||||
if newer := b.supersededBy(module, key, m.Fp2); newer != "" {
|
||||
whyEnded = b.saySuperseded(module, key, m.Fp2, newer)
|
||||
return
|
||||
}
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
var noModule *NoBuildModule
|
||||
if errors.As(planErr, &noModule) {
|
||||
@@ -680,6 +729,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
|
||||
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
|
||||
if trunk && b.originOf(key).First && planErr == nil && !nothing && !reserving {
|
||||
// A done build whose digests its end did not record is recorded at the next look (design 47 H2 H-A-492).
|
||||
if _, err := b.RecordAgain(ctx, module, m.Fp2, "at a look"); err != nil {
|
||||
b.logf("%v", err)
|
||||
}
|
||||
}
|
||||
if reserving && b.waitFill(module, m) {
|
||||
end("is not built yet: wait-held, the catalog has not said versions-filled")
|
||||
return
|
||||
@@ -727,7 +782,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
}
|
||||
var reserved versions.ReservedVersion
|
||||
hasReserved := false
|
||||
hasReserved, supersededNow := false, ""
|
||||
if rebuild && cl.Build.State == builds.BuildBuilding {
|
||||
// Reserved as A-490b amended by H-A-225 has it: this fingerprint's standing reservation is kept while it is unreleased
|
||||
// and above the highest the catalog's versions answers, else the next above that highest is taken (A-455); the build's
|
||||
@@ -737,10 +792,16 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
reserving = true
|
||||
} else if reserving && cl.Build.State != builds.BuildDone {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
// Looked at again just before it reserves (H-A-489): a newer entry heard since the claim is not taken over.
|
||||
if supersededNow = b.supersededBy(module, key, m.Fp2); supersededNow == "" {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
}
|
||||
}
|
||||
var result builds.BuilderBuild
|
||||
if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
if supersededNow != "" {
|
||||
// Ended as the newer entry's take-in withdraws a build in flight: final for this fingerprint.
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + supersededWhy(module, m.Fp2, supersededNow)})}
|
||||
} else if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
// A trunk build that reserves is never built without its version: it would end done and publish nothing, and its key would
|
||||
// stay done, never built again (A-490). The key fails with why and is taken again at the next look or by builder_build_again.
|
||||
result = failed("no version is reserved for this build: the builder's journal says why, and it is built again once one can be")
|
||||
@@ -765,7 +826,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
var w buildrun.Withdrawn
|
||||
if ctx.Err() == nil && errors.As(context.Cause(run), &w) {
|
||||
// Taken out, not stopped: the key says so, and the asks are withdrawn (the asker dropped them).
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{"withdrawn: " + w.Why})}
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + w.Why})}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
@@ -807,8 +868,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
for _, a := range result.Artifacts {
|
||||
digests[module+":"+a.Name] = a.Digest
|
||||
}
|
||||
if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err)
|
||||
// Under the builder's own bound, never the build's context: a takeover or an end that cancels it no longer leaves a passed
|
||||
// build unrecorded, so unreleased (design 47 H2 H-A-492).
|
||||
rec, stop := b.recordContext(ctx)
|
||||
err := b.Reserved.Built(rec, module, reserved.Version, digests, b.takeDependencies(key))
|
||||
stop()
|
||||
if err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v; recorded again at the builder's next look or start", cl.Key, reserved.Version, err)
|
||||
}
|
||||
}
|
||||
if result.State == builds.BuildFailed {
|
||||
|
||||
@@ -333,8 +333,9 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
asked, release := make(chan struct{}), make(chan struct{})
|
||||
answer := r.answer
|
||||
var once sync.Once
|
||||
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
||||
close(asked)
|
||||
once.Do(func() { close(asked) })
|
||||
<-release
|
||||
return answer(a)
|
||||
}
|
||||
@@ -356,6 +357,85 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "withdrawn: its reservation 0.1.3 was taken over") {
|
||||
t.Fatalf("the key = %+v, want failed withdrawn naming the take-over", v)
|
||||
}
|
||||
// H-A-489: withdrawn is final for the fingerprint: a later look builds it not again; a person's build again does.
|
||||
n, rev := len(r.asks.Asks()), r.st.revs[r.key]
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); len(r.asks.Asks()) != n || r.st.revs[r.key] != rev || !strings.HasPrefix(v.Error, "withdrawn: ") {
|
||||
t.Errorf("a later look built the withdrawn key again: %d asks (was %d), the key %+v", len(r.asks.Asks()), n, v)
|
||||
}
|
||||
if cl, err := r.b.Claimer.Claim(context.Background(), "m", fp1, mdigest('a'), ClaimOptions{ByHand: true}); err != nil || cl == nil {
|
||||
t.Errorf("a person's build again of the withdrawn key was not claimed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-489 (review round 1): a newer entry of the first branch heard between the older build's claim and its reservation is
|
||||
// not taken over: the older key ends failed withdrawn with no reservation, and the newer reserves.
|
||||
func TestANewerEntryHeardBeforeTheReservationIsNotTakenOver(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
newer, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
Fp2: fp2, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.st.before = func() { r.b.Module(context.Background(), "m", newer) } // at the older key's claim
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, "withdrawn: ") || !strings.Contains(v.Error, "superseded by the fingerprint "+fp2) {
|
||||
t.Errorf("the older key = %+v, want failed withdrawn, superseded by %s", v, fp2)
|
||||
}
|
||||
if len(res.sources) != 1 || res.sources[0].Fingerprint != fp2 {
|
||||
t.Errorf("reserved %+v, want one reservation, the newer's", res.sources)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-489: a fingerprint of the first branch that is no longer its newest entry heard is neither reserved for nor built, and
|
||||
// said once; the again loop that looks at it later builds nothing.
|
||||
func TestAnOlderFingerprintAskedAfterANewerEntryIsRefused(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
r.b.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
r.bm.fails["ace/build_go_build"] = "go build of tool: boom"
|
||||
older := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: fp1, Manifest: mdigest('a'),
|
||||
Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
delete(r.bm.fails, "ace/build_go_build")
|
||||
newer := older
|
||||
newer.Fp2 = fp2
|
||||
data, err := meshcatalogclient.EncodeModule(newer)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
r.b.Wait()
|
||||
rev := r.st.revs[r.key]
|
||||
for range 2 {
|
||||
r.b.consider(context.Background(), "m", older) // as consider's again loop or a step's look takes it
|
||||
r.b.Wait()
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || r.st.revs[r.key] != rev {
|
||||
t.Errorf("the older key = %+v at revision %d (was %d), want it left as it failed", v, r.st.revs[r.key], rev)
|
||||
}
|
||||
k2, _ := builds.BuilderBuildKey("m", fp2)
|
||||
if v := valueOf(t, r.st, k2); v.State != builds.BuildDone {
|
||||
t.Errorf("the newer key = %+v, want done", v)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range said {
|
||||
if strings.Contains(l, fp1+": is not built: superseded by "+fp2) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("superseded said %d times, want once: %q", n, said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key
|
||||
|
||||
@@ -142,6 +142,10 @@ func (b *Builder) buildEntry(ctx context.Context, pipeline string, e meshbuilds.
|
||||
}
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, failedWords, "existing build")
|
||||
}
|
||||
if newer := b.supersededBy(entry.Module, key, entry.Fp); newer != "" {
|
||||
// Refused, not waited on (H-A-489): no look builds a superseded fingerprint, so its end would never come.
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s %s", entry.Module, entry.Fp, b.saySuperseded(entry.Module, key, entry.Fp, newer)), "refused")
|
||||
}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
|
||||
b.consider(ctx, entry.Module, m)
|
||||
@@ -232,7 +236,7 @@ func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed str
|
||||
switch {
|
||||
case v.State == builds.BuildDone && !Unpublished(v, b.Seats):
|
||||
return true, "", true
|
||||
case v.State == builds.BuildFailed && builds.FailedUntilChanged(v.Error):
|
||||
case v.State == builds.BuildFailed && (builds.FailedUntilChanged(v.Error) || strings.HasPrefix(v.Error, withdrawnPrefix)):
|
||||
return false, key + ": " + v.Error, true
|
||||
}
|
||||
return false, "", false
|
||||
|
||||
@@ -74,6 +74,21 @@ func TestABuildStepAnswersItsEnd(t *testing.T) {
|
||||
if end := ask(entries...); end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Step != meshbuilds.StepEndedStepBuild || len(end.Entries) != 1 || end.Entries[0].Commit != "c1" || end.Pipeline != "main.1.m" {
|
||||
t.Errorf("a done build: %+v", end)
|
||||
}
|
||||
// H-A-489 (review round 1): a withdrawn key whose fingerprint is still the newest entry is answered failed at once, naming
|
||||
// withdrawn: no look builds it again, so its end is never waited for.
|
||||
rw := newRig(t, catalogue.Artifact{Name: "files", Kind: "archive"})
|
||||
rw.b.StepEntry = r.b.StepEntry
|
||||
withdrawn, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Manifest: mdigest('a'), Error: withdrawnPrefix + "its reservation 0.1.3 was taken over"})
|
||||
if _, err := rw.st.Create(t.Context(), rw.key, withdrawn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rw.b.Module(t.Context(), "m", moduleValue(t, "modules/m")) // the newest entry heard is fp1
|
||||
rw.b.Wait()
|
||||
bounded, cancel := context.WithTimeout(t.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
if end := rw.b.BuildStep(bounded, "ask1", meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries}); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "withdrawn: ") {
|
||||
t.Errorf("a withdrawn key asked: %+v, want failed at once naming withdrawn", end)
|
||||
}
|
||||
}
|
||||
|
||||
// A step ask waits for the end of its build without a timer: the log entry of that end wakes it (design 47 G A-97).
|
||||
@@ -434,9 +449,10 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved
|
||||
// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs;
|
||||
// a kept revision naming another fingerprint fails the step, naming it.
|
||||
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, while the older pipeline
|
||||
// asks the build of revision 45. Its kept catalog revision is read by number and the build runs while the picture never heard
|
||||
// a newer entry of m (a restart); once it heard one on the first branch, H-A-489 overrides H-A-205 and the older ask is refused
|
||||
// superseded. A kept revision naming another fingerprint fails the step, naming it.
|
||||
func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
var held atomic.Int64
|
||||
held.Store(46)
|
||||
@@ -460,6 +476,24 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 {
|
||||
t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources)
|
||||
}
|
||||
// H-A-489: once the picture heard the newer entry of m on the first branch, the older ask is refused superseded and
|
||||
// nothing is built or reserved for it; the newer entry's build keeps the series.
|
||||
rn, resn, entryn := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
rn.b.RevisionWait = time.Nanosecond
|
||||
rn.b.FirstBranch = r.b.FirstBranch
|
||||
rn.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
return 45, map[string]string{"m": fp1}, nil
|
||||
}
|
||||
entryn.Fp = fp2
|
||||
rn.b.Entry(context.Background(), Origin{Branch: "main", Source: 46, First: true}, entryn)
|
||||
rn.b.Wait()
|
||||
if endn := rn.b.BuildStep(t.Context(), "a1", run); endn.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(endn.Said, "superseded by "+fp2) {
|
||||
t.Errorf("the older ask after the newer entry: end = %+v; want failed, superseded by %s", endn, fp2)
|
||||
}
|
||||
rn.b.Wait()
|
||||
if _, built := rn.st.vals[rn.key]; built || len(resn.sources) != 1 || resn.sources[0].Fingerprint != fp2 {
|
||||
t.Errorf("built %v, reserved %+v; want nothing built for the older, one reservation, the newer's", built, resn.sources)
|
||||
}
|
||||
r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
r2.b.RevisionWait = time.Nanosecond
|
||||
r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
|
||||
@@ -10,10 +10,8 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -42,8 +40,8 @@ const (
|
||||
// VersionHeldElsewhere is the finding of a number a registry seat already holds, skipped at reserving or moved from (design 47
|
||||
// H H-A-275), scope module.
|
||||
VersionHeldElsewhere = "version-held-elsewhere"
|
||||
// VersionTaggedElsewhere is the finding of a number whose git tag stands at another commit, skipped at reserving (design 47
|
||||
// H2 §796), scope module; cleared at the module's next version-released.
|
||||
// VersionTaggedElsewhere is the finding of a number whose git tag stood at another commit, which a reserve skipped before no
|
||||
// tag was read (design 47 H2 H-A-494); one raised before is cleared at the module's next version-released.
|
||||
VersionTaggedElsewhere = "version-tagged-elsewhere"
|
||||
)
|
||||
|
||||
@@ -55,6 +53,10 @@ type ReservedVersion struct {
|
||||
Version string `json:"version"` // MAJOR.MINOR.PATCH
|
||||
// Artifacts are the module's artifacts at the version: the digest once built, empty before.
|
||||
Artifacts map[string]string `json:"artifacts"`
|
||||
// Upstream are the artifacts of kind upstream, by reference (design 47 H2 H-A-486): kept in Artifacts with their digest for the
|
||||
// record, never awaited, as design 51 copies an upstream image by its pinned digest and no registry seat says it published at
|
||||
// a version. A reservation written before carries none.
|
||||
Upstream []string `json:"upstream,omitempty"`
|
||||
// BuiltAt is when the build ended with every digest known; the publish's bound counts from it.
|
||||
BuiltAt time.Time `json:"built-at,omitzero"`
|
||||
// Published are the artifacts a registry seat said published at this version with the built digest.
|
||||
@@ -63,8 +65,8 @@ type ReservedVersion struct {
|
||||
// was kept.
|
||||
PublishedAt map[string]time.Time `json:"published-at,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go
|
||||
// package's, GoTagOf): empty for a module with no Go package (design 47 G §370), or whose Go package's directory is not known.
|
||||
// Repository and Commit are what the build was made from. Tag is kept for a reservation written before design 47 H2 H-A-494:
|
||||
// no git tag is written at a release any more, and a new reservation names none.
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
@@ -110,22 +112,19 @@ type KeptPublished struct {
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
|
||||
// written at, and whether the module has a Go package, whose version is tagged.
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit, kept on the reservation.
|
||||
type Source struct {
|
||||
Repository, Commit string
|
||||
// Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76).
|
||||
Fingerprint string
|
||||
GoPackage bool
|
||||
// GoArtifact is the artifact that is the module's Go package, whose module path names the directory its tag is written in
|
||||
// (design 47 H2 §796).
|
||||
GoArtifact string
|
||||
// Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of
|
||||
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
|
||||
Branch string
|
||||
Revision int64
|
||||
// CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did.
|
||||
CatalogRevision int64
|
||||
// Upstream are the artifacts of kind upstream among those reserved, by reference (H-A-486), kept on the reservation.
|
||||
Upstream []string
|
||||
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
|
||||
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
|
||||
Rebuild bool
|
||||
@@ -171,92 +170,6 @@ type Reserver struct {
|
||||
HeldNumbers func(ctx context.Context, module, fingerprint string, refs []string) ([]string, error)
|
||||
// HeldElsewhere is told the numbers of module a reserve skipped because a registry holds them (H-A-275); nil is told none.
|
||||
HeldElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
// Names answers a Go package's module path, which names the directory its tag is written in (design 47 H2 §796); nil, or a
|
||||
// name not known, tags the version nowhere, said.
|
||||
Names PackageNames
|
||||
// Tags reads repository's tags starting with prefix, each with the commit it names, through the seat git's tags (design 47
|
||||
// H2 §796); away says the read met a transport fault or the error kind away, which lets the reservation go on, said. Nil
|
||||
// reads none.
|
||||
Tags func(ctx context.Context, repository, prefix string) (tags []git.Tag, away bool, err error)
|
||||
// TaggedElsewhere is told the numbers of module a reserve skipped because their tag stands at another commit (§796); nil
|
||||
// is told none.
|
||||
TaggedElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
|
||||
tagsMu sync.Mutex
|
||||
// tagsRead are the tags last read of each module, by series patch to the commit, with the catalog revision they were read
|
||||
// under: a module's tags are read once per catalog revision it reserves under (§796).
|
||||
tagsRead map[string]tagsAt
|
||||
}
|
||||
|
||||
// tagsAt are a module's tags of one series read under one catalog revision: patch to the commit its tag names.
|
||||
type tagsAt struct {
|
||||
revision, prefix string
|
||||
commits map[int]string
|
||||
}
|
||||
|
||||
// goDirectory is the directory of module's Go package in its repository (§796), from its module path; ok is false, said, when
|
||||
// no name or no directory of the repository is known.
|
||||
func (r *Reserver) goDirectory(ctx context.Context, module string, src Source) (string, bool) {
|
||||
if r.Names == nil {
|
||||
return "", false
|
||||
}
|
||||
path, known, err := r.Names(ctx, module, src.GoArtifact)
|
||||
switch {
|
||||
case err != nil:
|
||||
r.said(fmt.Errorf("%s: its Go package's name cannot be read, so its version is tagged nowhere: %w", module, err))
|
||||
return "", false
|
||||
case !known:
|
||||
r.said(fmt.Errorf("%s: no name of its Go package %s is known yet, so its version is tagged nowhere", module, src.GoArtifact))
|
||||
return "", false
|
||||
}
|
||||
dir, ok := GoDirectory(path, src.Repository)
|
||||
if !ok {
|
||||
r.said(fmt.Errorf("%s: its Go package %s is not a module path of the repository %s, so its version is tagged nowhere", module, path, src.Repository))
|
||||
}
|
||||
return dir, ok
|
||||
}
|
||||
|
||||
// taggedAt are the patches of series whose tag `<directory>/v<series>.<patch>` stands, each with its commit, read once per
|
||||
// catalog revision the module reserves under (§796). A read that meets a transport fault or away answers none, said, and the
|
||||
// reservation goes on; any other failure refuses the reservation, naming it.
|
||||
func (r *Reserver) taggedAt(ctx context.Context, module, series, dir string, src Source) (map[int]string, error) {
|
||||
if r.Tags == nil {
|
||||
return nil, nil
|
||||
}
|
||||
prefix := TagPrefix(dir) + series + "."
|
||||
revision := ""
|
||||
if src.CatalogRevision > 0 {
|
||||
revision = fmt.Sprintf("%s.%d", src.Branch, src.CatalogRevision)
|
||||
r.tagsMu.Lock()
|
||||
got, ok := r.tagsRead[module]
|
||||
r.tagsMu.Unlock()
|
||||
if ok && got.revision == revision && got.prefix == prefix {
|
||||
return got.commits, nil
|
||||
}
|
||||
}
|
||||
tags, away, err := r.Tags(ctx, src.Repository, prefix)
|
||||
if err != nil && away {
|
||||
r.said(fmt.Errorf("%s: its tags %s* cannot be read (%w), so the reservation goes on without them", module, prefix, err))
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the tags %s* of %s cannot be read: %w", prefix, module, err)
|
||||
}
|
||||
commits := map[int]string{}
|
||||
for _, t := range tags {
|
||||
if s, p, err := SplitVersion(strings.TrimPrefix(t.Tag, TagPrefix(dir))); err == nil && s == series && strings.HasPrefix(t.Tag, prefix) {
|
||||
commits[p] = t.Commit
|
||||
}
|
||||
}
|
||||
if revision != "" {
|
||||
r.tagsMu.Lock()
|
||||
if r.tagsRead == nil {
|
||||
r.tagsRead = map[string]tagsAt{}
|
||||
}
|
||||
r.tagsRead[module] = tagsAt{revision: revision, prefix: prefix, commits: commits}
|
||||
r.tagsMu.Unlock()
|
||||
}
|
||||
return commits, nil
|
||||
}
|
||||
|
||||
// heldNumbers are the patches of series a registry holds of module's artifacts (H-A-275); a read that fails is said, and none.
|
||||
@@ -277,19 +190,15 @@ func (r *Reserver) heldNumbers(ctx context.Context, module, series, fingerprint
|
||||
return out
|
||||
}
|
||||
|
||||
// nextFree is the first patch after after that no registry holds and whose tag stands at no other commit than commit, with the
|
||||
// held numbers it skipped (H-A-275) and those tagged elsewhere (design 47 H2 §796).
|
||||
func nextFree(series string, after int, held map[int]bool, tagged map[int]string, commit string) (int, []string, []string) {
|
||||
var skipped, elsewhere []string
|
||||
// nextFree is the first patch after after that no registry holds, with the held numbers it skipped (H-A-275). No git tag is
|
||||
// read: a series is numbered from the catalog's versions and the registries' alone (design 47 H2 H-A-494).
|
||||
func nextFree(series string, after int, held map[int]bool) (int, []string) {
|
||||
var skipped []string
|
||||
for p := after + 1; ; p++ {
|
||||
switch c, ok := tagged[p]; {
|
||||
case held[p]:
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
case ok && c != commit:
|
||||
elsewhere = append(elsewhere, fmt.Sprintf("%s.%d (tagged at %s)", series, p, c))
|
||||
default:
|
||||
return p, skipped, elsewhere
|
||||
if !held[p] {
|
||||
return p, skipped
|
||||
}
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,15 +251,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
}
|
||||
key := ReservedKey(module, series)
|
||||
held := r.heldNumbers(ctx, module, series, src.Fingerprint, artifacts)
|
||||
dir, tagged := "", false
|
||||
var tags map[int]string
|
||||
if src.GoPackage {
|
||||
if dir, tagged = r.goDirectory(ctx, module, src); tagged {
|
||||
if tags, err = r.taggedAt(ctx, module, series, dir, src); err != nil {
|
||||
return ReservedVersion{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
for {
|
||||
cur, rev, found, err := r.get(ctx, key)
|
||||
if err != nil {
|
||||
@@ -368,15 +268,13 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
next = p
|
||||
}
|
||||
}
|
||||
patch, skipped, elsewhere := nextFree(series, next, held, tags, src.Commit)
|
||||
patch, skipped := nextFree(series, next, held)
|
||||
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, patch), Artifacts: map[string]string{}, At: r.now(),
|
||||
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
|
||||
if tagged {
|
||||
out.Tag = GoTagOf(dir, out.Version)
|
||||
}
|
||||
for _, a := range artifacts {
|
||||
out.Artifacts[a] = ""
|
||||
}
|
||||
out.Upstream = slices.Clone(src.Upstream)
|
||||
if found {
|
||||
out.Withdrawn = maps.Clone(cur.Withdrawn)
|
||||
if cur.Fingerprint != src.Fingerprint {
|
||||
@@ -410,9 +308,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
if len(skipped) > 0 && r.HeldElsewhere != nil {
|
||||
r.HeldElsewhere(ctx, module, skipped)
|
||||
}
|
||||
if len(elsewhere) > 0 && r.TaggedElsewhere != nil {
|
||||
r.TaggedElsewhere(ctx, module, elsewhere)
|
||||
}
|
||||
if found && r.Took != nil {
|
||||
r.Took(ctx, cur, out)
|
||||
}
|
||||
@@ -1134,11 +1029,11 @@ const (
|
||||
ReservedVerb = meshbuilds.VerbReserved
|
||||
)
|
||||
|
||||
// Missing are the artifacts of v no registry seat said published, sorted.
|
||||
// Missing are the artifacts of v no registry seat said published, sorted; an upstream one is never awaited (H-A-486).
|
||||
func (v ReservedVersion) Missing() []string {
|
||||
var out []string
|
||||
for a := range v.Artifacts {
|
||||
if !v.Published[a] {
|
||||
if !v.Published[a] && !slices.Contains(v.Upstream, a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
)
|
||||
|
||||
@@ -171,6 +169,23 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
if got := cur.Missing(); !reflect.DeepEqual(got, []string{"m:b"}) {
|
||||
t.Errorf("missing = %v, want m:b", got)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved and recorded with its digest, but never awaited: once the others are published
|
||||
// nothing is missing.
|
||||
up, err := r.Reserve(ctx, "u", "0.1", []string{"u:server", "u:tools"}, Source{Upstream: []string{"u:server"}})
|
||||
if err != nil || !reflect.DeepEqual(up.Upstream, []string{"u:server"}) {
|
||||
t.Fatalf("reserved %+v (%v), want u:server kept as upstream", up, err)
|
||||
}
|
||||
for _, a := range []string{"u:server", "u:tools"} {
|
||||
if _, err := r.Built(ctx, "u", up.Version, a, "sha256:"+a, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if marked, _, err := r.Published(ctx, "u-tools", up.Version, "sha256:u:tools"); err != nil || len(marked) != 1 {
|
||||
t.Fatalf("the published of u:tools marked %v (%v)", marked, err)
|
||||
}
|
||||
if cur, _, _ := r.At(ctx, ReservedKey("u", "0.1")); len(cur.Missing()) != 0 || cur.Artifacts["u:server"] != "sha256:u:server" {
|
||||
t.Errorf("missing = %v with artifacts %v, want none missing and the upstream digest kept", cur.Missing(), cur.Artifacts)
|
||||
}
|
||||
if key, ok := ReservedBoundOf(ReservedBoundKey("m", "1.4")); !ok || key != ReservedKey("m", "1.4") || ReservedBoundFor(key) != ReservedBoundKey("m", "1.4") {
|
||||
t.Errorf("the bound key does not name the reservation back: %q %v", key, ok)
|
||||
}
|
||||
@@ -399,7 +414,7 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
return held, readErr
|
||||
},
|
||||
HeldElsewhere: func(_ context.Context, _ string, s []string) { skipped = append(skipped, s...) }}
|
||||
v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true})
|
||||
v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1"})
|
||||
if err != nil || v.Version != "0.1.39" || len(skipped) != 2 {
|
||||
t.Fatalf("reserved %q (%v), skipped %v: want 0.1.39 past the two numbers the registry holds", v.Version, err, skipped)
|
||||
}
|
||||
@@ -413,9 +428,9 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
}
|
||||
held = []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40", "0.2.0"}
|
||||
// The build again of a done key reserves as a rebuild (A-490b): a withdrawn number is never its floor's to reuse.
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true, Rebuild: true})
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1", Rebuild: true})
|
||||
if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != "" {
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (no Go package name known)", again, err)
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (design 47 H2 H-A-494)", again, err)
|
||||
}
|
||||
if at, _ := r.AtVersion(ctx, "0.1.39"); len(at) != 0 {
|
||||
t.Errorf("the index by version still names the withdrawn number: %v", at)
|
||||
@@ -428,50 +443,4 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
if v, err := r.Reserve(ctx, "node-tools", "0.2", []string{"node-tools:program"}, Source{Fingerprint: "fp1"}); err != nil || v.Version != "0.2.1" {
|
||||
t.Errorf("with the registry down reserved %q (%v), want 0.2.1 (what it answered still skipped)", v.Version, err)
|
||||
}
|
||||
// Design 47 H2 §796: a Go package's tags are read before it reserves, once per catalog revision; a number whose tag
|
||||
// `<directory>/v<version>` stands at another commit is skipped and said, one at the build's own commit is not; the version's
|
||||
// tag names the Go module's directory; a read met away lets the reservation go on, any other failure refuses it.
|
||||
r.HeldNumbers = nil
|
||||
var tagReads int
|
||||
var tagsErr error
|
||||
var tagsAway bool
|
||||
var elsewhere []string
|
||||
tags := []git.Tag{{Tag: "modules/nt/v0.3.0", Commit: "c-other"}, {Tag: "modules/nt/v0.3.1", Commit: "c-mine"}, {Tag: "modules/nt/v0.3.2", Commit: "c-other2"}}
|
||||
r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/nt", module == "node-tools" && artifact == "lib", nil
|
||||
}
|
||||
r.Tags = func(_ context.Context, repository, prefix string) ([]git.Tag, bool, error) {
|
||||
tagReads++
|
||||
if repository != "o/r" || prefix != "modules/nt/v0.3." {
|
||||
t.Errorf("tags read of %s %q, want o/r modules/nt/v0.3.", repository, prefix)
|
||||
}
|
||||
return tags, tagsAway, tagsErr
|
||||
}
|
||||
r.TaggedElsewhere = func(_ context.Context, _ string, s []string) { elsewhere = append(elsewhere, s...) }
|
||||
at := Source{Fingerprint: "fp3", GoPackage: true, GoArtifact: "lib", Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 7}
|
||||
v3, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at)
|
||||
if err != nil || v3.Version != "0.3.1" || v3.Tag != "modules/nt/v0.3.1" || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.0") {
|
||||
t.Fatalf("reserved %+v (%v), tagged elsewhere %v: want 0.3.1 (0.3.0 tagged at another commit, 0.3.1 at the build's own), tag modules/nt/v0.3.1", v3, err, elsewhere)
|
||||
}
|
||||
at.Fingerprint, at.Commit, elsewhere = "fp4", "c-new", nil
|
||||
if v4, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v4.Version != "0.3.3" || tagReads != 1 || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.2") {
|
||||
t.Fatalf("reserved %+v (%v) after %d reads, tagged elsewhere %v: want 0.3.3 past 0.3.2's tag, the tags read once for revision main.7", v4, err, tagReads, elsewhere)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsErr, tagsAway = 8, "fp5", errors.New("the forge is away"), true
|
||||
if v5, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v5.Version != "0.3.4" || tagReads != 2 {
|
||||
t.Fatalf("with the tags away reserved %+v (%v) after %d reads, want 0.3.4 and the reservation going on", v5, err, tagReads)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsAway = 9, "fp6", false
|
||||
if _, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err == nil || !strings.Contains(err.Error(), "tags") {
|
||||
t.Errorf("a tags read refused not away reserved anyway (%v), want the reservation refused naming the tags", err)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/novox/mesh-sdk/go", "https://git.example/novox/mesh-sdk.git"); !ok || d != "go" || GoTagOf(d, "0.1.9") != "go/v0.1.9" {
|
||||
t.Errorf("mesh-sdk's Go package: directory %q (%v), want go and the tag go/v0.1.9", d, ok)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/o/r", "o/r"); !ok || d != "" || GoTagOf(d, "1.0.0") != "v1.0.0" {
|
||||
t.Errorf("a root Go module: directory %q (%v), want the root and the tag v1.0.0", d, ok)
|
||||
}
|
||||
if _, ok := GoDirectory("git.example/o/rr/x", "o/r"); ok {
|
||||
t.Error("a module path of another repository was given a directory")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package versions
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// GoTagOf is the git tag a Go package's released version is written as (design 47 H2 §796): `<directory>/v<version>`, the
|
||||
// directory of its Go module in the repository, which the Go toolchain's direct fallback reads; `v<version>` at the root. The
|
||||
// `go/v<version>` tags go from new versions.
|
||||
func GoTagOf(directory, version string) string { return TagPrefix(directory) + version }
|
||||
|
||||
// GoDirectory is the directory of the Go module path modulePath in repository (`<owner>/<name>`, or a clone address naming
|
||||
// it): the part of the path after the repository's own, as the Go toolchain reads a module kept in a subdirectory; empty at
|
||||
// the root. ok is false when the path is not one of that repository.
|
||||
func GoDirectory(modulePath, repository string) (string, bool) {
|
||||
repo := "/" + strings.ToLower(trunk.RepositoryName(repository))
|
||||
lower := strings.ToLower(modulePath)
|
||||
for from := 0; ; {
|
||||
i := strings.Index(lower[from:], repo)
|
||||
if i < 0 || repo == "/" {
|
||||
return "", false
|
||||
}
|
||||
end := from + i + len(repo)
|
||||
if end == len(lower) || lower[end] == '/' {
|
||||
return strings.Trim(modulePath[end:], "/"), true
|
||||
}
|
||||
from = end
|
||||
}
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
// Design 51 *The floor*: the floor's three sources, the registry seats' versions, git.tags and indexer_requirements.
|
||||
// The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3).
|
||||
if want := []string{"seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
|
||||
"seat:git.tag", "seat:git.tags",
|
||||
"seat:git.tags", // the floor's source alone: no tag is written at a release (design 47 H2 H-A-494)
|
||||
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
|
||||
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions",
|
||||
// A-231: the build asks on each build seat's queue delivery (accept.delivery.<asker>).
|
||||
@@ -199,15 +199,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 {
|
||||
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
|
||||
}
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 ||
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 13 ||
|
||||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
|
||||
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
|
||||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime ||
|
||||
// H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default
|
||||
m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime ||
|
||||
m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
// H-A-492: the record of a version's digests runs under the builder's own bound, a setting of 60 s by default
|
||||
m.Config["record-bound-seconds"].Default != float64(60) ||
|
||||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 {
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), record-bound-seconds of 60 (H-A-492), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
}
|
||||
// Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends.
|
||||
if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
@@ -266,7 +268,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("health = %v, want kind unit alone", p["health"])
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
if len(env) != 9 || env["MESH_BUILDER_RECORD_BOUND_FILE"] != "${dir:state}/record-bound-seconds" || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
|
||||
}
|
||||
for k, v := range env {
|
||||
@@ -274,14 +276,16 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("env %s = %v: a setting reaches the process as a file, never its environment (A-71)", k, v)
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "file" && r["path"] == "${dir:state}/toolchain-wait-minutes" && r["content"] == "${config:toolchain-wait-minutes}" {
|
||||
found = true
|
||||
for _, setting := range []string{"toolchain-wait-minutes", "record-bound-seconds"} { // H-A-492: the record's bound too
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "file" && r["path"] == "${dir:state}/"+setting && r["content"] == "${config:"+setting+"}" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no file resource with the content ${config:%s}: A-71", setting)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no file resource with the content ${config:toolchain-wait-minutes}: A-71")
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "settings" {
|
||||
|
||||
@@ -91,7 +91,6 @@
|
||||
"invokes": [
|
||||
"seat:container-image-registry.remove",
|
||||
"seat:container-image-registry.versions",
|
||||
"seat:git.tag",
|
||||
"seat:git.tags",
|
||||
"seat:go-package-registry.remove",
|
||||
"seat:go-package-registry.versions",
|
||||
@@ -183,6 +182,14 @@
|
||||
"trusted": false,
|
||||
"default": 10
|
||||
},
|
||||
"record-bound-seconds": {
|
||||
"type": "number",
|
||||
"description": "seconds the builder gives the record of a version's digests after its build ends, under its own bound and never the build's (design 47 H2 H-A-492).",
|
||||
"minimum": 1,
|
||||
"maximum": 3600,
|
||||
"trusted": false,
|
||||
"default": 60
|
||||
},
|
||||
"ask-taken-bound": {
|
||||
"type": "string",
|
||||
"description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).",
|
||||
@@ -310,6 +317,14 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:package-wait-minutes}"
|
||||
},
|
||||
{
|
||||
"id": "record-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/record-bound-seconds",
|
||||
"owner": "mesh-delivery-builder",
|
||||
"mode": "0600",
|
||||
"content": "${config:record-bound-seconds}"
|
||||
},
|
||||
{
|
||||
"id": "publish-pause",
|
||||
"type": "file",
|
||||
@@ -361,6 +376,7 @@
|
||||
"MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes",
|
||||
"MESH_BUILDER_REVISION_WAIT_FILE": "${dir:state}/revision-wait-minutes",
|
||||
"MESH_BUILDER_PACKAGE_WAIT_FILE": "${dir:state}/package-wait-minutes",
|
||||
"MESH_BUILDER_RECORD_BOUND_FILE": "${dir:state}/record-bound-seconds",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_FILE": "${dir:state}/publish-pause",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE": "${dir:state}/publish-pause-max",
|
||||
"MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries",
|
||||
|
||||
@@ -14,13 +14,13 @@ const Seat = "mesh-pipeline-runner"
|
||||
|
||||
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
|
||||
// a reader of a value of another form says "form N, this reader knows M".
|
||||
const Form = 8
|
||||
const Form = 10
|
||||
|
||||
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
|
||||
// whose fields changed while its forms did not (design 47 H2 §780).
|
||||
const (
|
||||
generatedForms = "8"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionAgainNValue=693580eed498 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
generatedForms = "10"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=e3f4c58f93c8 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=582f8ff3b28d PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionAgainNValue=693580eed498 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -775,7 +775,7 @@ type Pipeline struct {
|
||||
Revision int64 `json:"revision,omitempty"`
|
||||
// the run on request's id, for a pipeline of a run; such a pipeline delivers nothing
|
||||
Run string `json:"run,omitempty"`
|
||||
// for a pipeline written over pipelines-at-once, its place in the queue: the runner's counter queue.next at its write (design 47 H §748)
|
||||
// its place in the runner's sequence: the runner's counter queue.next at its write, taken for every pipeline it writes, queued or placed at once, so it grows across the pipelines of a branch (design 47 H §748; H2 §813 (1), H-A-504 addendum)
|
||||
Seq int64 `json:"seq,omitempty"`
|
||||
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
|
||||
Source int64 `json:"source"`
|
||||
@@ -897,6 +897,8 @@ type PipelineEnded struct {
|
||||
Revision int64 `json:"revision,omitempty"`
|
||||
// the run on request's id, for a pipeline of a run; such a pipeline delivers nothing
|
||||
Run string `json:"run,omitempty"`
|
||||
// the pipeline's sequence, its record's seq: the assigner writes it as the run of the wants the pipeline's pass writes (design 47 H2 §813 (1), H-A-504 addendum)
|
||||
Seq int64 `json:"seq,omitempty"`
|
||||
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
|
||||
Source int64 `json:"source"`
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
@@ -948,7 +950,7 @@ const (
|
||||
PipelineRowOutcomeCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// PipelineStarted is a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52).
|
||||
// PipelineStarted is a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52), and again at a re-pin with its kept revisions (H2 H-A-496).
|
||||
type PipelineStarted struct {
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
// the watched branch, or the branch of a run on request
|
||||
@@ -957,6 +959,8 @@ type PipelineStarted struct {
|
||||
Call string `json:"call,omitempty"`
|
||||
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
|
||||
Form int64 `json:"form,omitempty"`
|
||||
// every catalog revision the pipeline's movers are kept at, said again in full at each re-pin, so the indexer pins each while the pipeline runs (design 47 H2 H-A-496 and its addendum)
|
||||
Kept []int64 `json:"kept,omitempty"`
|
||||
// the pipeline's movers
|
||||
Movers []Mover `json:"movers"`
|
||||
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
|
||||
|
||||
@@ -19,6 +19,10 @@ func TestFixtures(t *testing.T) {
|
||||
}{
|
||||
{name: "a take-in's pipeline started with its two movers", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.3589", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"repository\":\"mesh-sdk\",\"tier\":1},{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":2}],\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"repository\":\"mesh-sdk\",\"tier\":1},{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":2}],\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineStartedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-started\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline started again at a re-pin, with the three revisions its movers are kept at (H-A-496)", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.291", given: "{\"at\":\"2026-11-22T20:00:00Z\",\"branch\":\"main\",\"kept\":[292,295,298],\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"pipeline\":\"main.291\",\"revision\":291,\"source\":114,\"v\":1}", wire: "{\"at\":\"2026-11-22T20:00:00Z\",\"branch\":\"main\",\"kept\":[292,295,298],\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"pipeline\":\"main.291\",\"revision\":291,\"source\":114,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineStartedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-started\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "an again pipeline ended passed, with its sequence (H-A-504 addendum)", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.320", given: "{\"branch\":\"main\",\"end-count\":1,\"movers\":[{\"fingerprint\":\"fp-0123\",\"module\":\"node-tools\",\"steps\":[],\"tier\":1,\"version\":\"0.1.115\"}],\"outcome\":\"passed\",\"pipeline\":\"main.320.again.1\",\"revision\":320,\"seq\":269,\"source\":119,\"v\":1}", wire: "{\"branch\":\"main\",\"end-count\":1,\"movers\":[{\"fingerprint\":\"fp-0123\",\"module\":\"node-tools\",\"steps\":[],\"tier\":1,\"version\":\"0.1.115\"}],\"outcome\":\"passed\",\"pipeline\":\"main.320.again.1\",\"revision\":320,\"seq\":269,\"source\":119,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-ended\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline ended passed: per mover its version, steps and tier, an optional step failed", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.3589", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-ended\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline of two movers ended with its phases and one mover left unfinished", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.3589", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"form\":2,\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"phases\":{\"build\":{\"end\":\"2026-11-21T22:02:00Z\",\"start\":\"2026-11-21T22:00:06Z\"},\"composition\":{\"end\":\"2026-11-21T22:06:00Z\",\"start\":\"2026-11-21T22:04:01Z\"},\"ended\":\"2026-11-21T22:06:00Z\",\"placed\":\"2026-11-21T22:00:05Z\",\"taken\":\"2026-11-21T22:00:00Z\",\"unit\":{\"end\":\"2026-11-21T22:04:00Z\",\"start\":\"2026-11-21T22:02:01Z\"}},\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"unfinished\":[\"mesh-delivery-view\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"form\":2,\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"phases\":{\"build\":{\"end\":\"2026-11-21T22:02:00Z\",\"start\":\"2026-11-21T22:00:06Z\"},\"composition\":{\"end\":\"2026-11-21T22:06:00Z\",\"start\":\"2026-11-21T22:04:01Z\"},\"ended\":\"2026-11-21T22:06:00Z\",\"placed\":\"2026-11-21T22:00:05Z\",\"taken\":\"2026-11-21T22:00:00Z\",\"unit\":{\"end\":\"2026-11-21T22:04:00Z\",\"start\":\"2026-11-21T22:02:01Z\"}},\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"unfinished\":[\"mesh-delivery-view\"],\"v\":1}",
|
||||
|
||||
@@ -29,6 +29,53 @@
|
||||
"at": "2026-11-10T10:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a pipeline started again at a re-pin, with the three revisions its movers are kept at (H-A-496)",
|
||||
"of": "event:pipeline-started",
|
||||
"subject": "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.291",
|
||||
"given": {
|
||||
"pipeline": "main.291",
|
||||
"branch": "main",
|
||||
"revision": 291,
|
||||
"source": 114,
|
||||
"v": 1,
|
||||
"movers": [
|
||||
{
|
||||
"module": "nats",
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"fingerprint": "fp-0123",
|
||||
"repository": "mesh-networking",
|
||||
"tier": 1
|
||||
}
|
||||
],
|
||||
"kept": [292, 295, 298],
|
||||
"at": "2026-11-22T20:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "an again pipeline ended passed, with its sequence (H-A-504 addendum)",
|
||||
"of": "event:pipeline-ended",
|
||||
"subject": "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.320",
|
||||
"given": {
|
||||
"pipeline": "main.320.again.1",
|
||||
"branch": "main",
|
||||
"revision": 320,
|
||||
"seq": 269,
|
||||
"source": 119,
|
||||
"end-count": 1,
|
||||
"v": 1,
|
||||
"outcome": "passed",
|
||||
"movers": [
|
||||
{
|
||||
"module": "node-tools",
|
||||
"fingerprint": "fp-0123",
|
||||
"steps": [],
|
||||
"tier": 1,
|
||||
"version": "0.1.115"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a pipeline ended passed: per mover its version, steps and tier, an optional step failed",
|
||||
"of": "event:pipeline-ended",
|
||||
|
||||
@@ -1357,7 +1357,7 @@
|
||||
},
|
||||
"seq": {
|
||||
"type": "integer",
|
||||
"description": "for a pipeline written over pipelines-at-once, its place in the queue: the runner's counter queue.next at its write (design 47 H §748)"
|
||||
"description": "its place in the runner's sequence: the runner's counter queue.next at its write, taken for every pipeline it writes, queued or placed at once, so it grows across the pipelines of a branch (design 47 H §748; H2 §813 (1), H-A-504 addendum)"
|
||||
},
|
||||
"end-count": {
|
||||
"type": "integer",
|
||||
@@ -1455,6 +1455,10 @@
|
||||
"type": "integer",
|
||||
"description": "the catalog graph revision of the branch the pipeline runs for"
|
||||
},
|
||||
"seq": {
|
||||
"type": "integer",
|
||||
"description": "the pipeline's sequence, its record's seq: the assigner writes it as the run of the wants the pipeline's pass writes (design 47 H2 §813 (1), H-A-504 addendum)"
|
||||
},
|
||||
"end-count": {
|
||||
"type": "integer",
|
||||
"description": "how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325)"
|
||||
@@ -1675,9 +1679,16 @@
|
||||
"published": {
|
||||
"$ref": "#/defs/Published",
|
||||
"description": "for a pipeline a package's publish started, that publish (A-202)"
|
||||
},
|
||||
"kept": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"description": "every catalog revision the pipeline's movers are kept at, said again in full at each re-pin, so the indexer pins each while the pipeline runs (design 47 H2 H-A-496 and its addendum)"
|
||||
}
|
||||
},
|
||||
"description": "a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52)",
|
||||
"description": "a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52), and again at a re-pin with its kept revisions (H2 H-A-496)",
|
||||
"required": [
|
||||
"v",
|
||||
"pipeline",
|
||||
@@ -2367,5 +2378,5 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"form": 8
|
||||
"form": 10
|
||||
}
|
||||
|
||||
@@ -306,7 +306,7 @@ export interface Pipeline {
|
||||
"revision"?: number;
|
||||
/** the run on request's id, for a pipeline of a run; such a pipeline delivers nothing */
|
||||
"run"?: string;
|
||||
/** for a pipeline written over pipelines-at-once, its place in the queue: the runner's counter queue.next at its write (design 47 H §748) */
|
||||
/** its place in the runner's sequence: the runner's counter queue.next at its write, taken for every pipeline it writes, queued or placed at once, so it grows across the pipelines of a branch (design 47 H §748; H2 §813 (1), H-A-504 addendum) */
|
||||
"seq"?: number;
|
||||
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
|
||||
"source": number;
|
||||
@@ -417,6 +417,8 @@ export interface PipelineEnded {
|
||||
"revision"?: number;
|
||||
/** the run on request's id, for a pipeline of a run; such a pipeline delivers nothing */
|
||||
"run"?: string;
|
||||
/** the pipeline's sequence, its record's seq: the assigner writes it as the run of the wants the pipeline's pass writes (design 47 H2 §813 (1), H-A-504 addendum) */
|
||||
"seq"?: number;
|
||||
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
|
||||
"source": number;
|
||||
"started"?: string;
|
||||
@@ -446,7 +448,7 @@ export interface PipelineRow {
|
||||
"steps"?: Record<string, number>;
|
||||
}
|
||||
|
||||
/** a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52) */
|
||||
/** a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52), and again at a re-pin with its kept revisions (H2 H-A-496) */
|
||||
export interface PipelineStarted {
|
||||
"at"?: string;
|
||||
/** the watched branch, or the branch of a run on request */
|
||||
@@ -455,6 +457,8 @@ export interface PipelineStarted {
|
||||
"call"?: string;
|
||||
/** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */
|
||||
"form"?: number;
|
||||
/** every catalog revision the pipeline's movers are kept at, said again in full at each re-pin, so the indexer pins each while the pipeline runs (design 47 H2 H-A-496 and its addendum) */
|
||||
"kept"?: number[];
|
||||
/** the pipeline's movers */
|
||||
"movers": Mover[];
|
||||
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
@@ -800,8 +801,10 @@ func serveAssignerAssignments(t *testing.T, ctx context.Context, js jetstream.Je
|
||||
}
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
// and the build seat's step-ended the manifest's consumes names (design 47 G §713: the end of a validation ask), and the
|
||||
// builder's version-reserved, which no stream of its own keeps (design 47 H2 §805).
|
||||
FilterSubjects: []string{"mesh.mod." + assignerHolderModule + ".event.views.>", meshcatalog.SubjectRoom, nodebuildgo.SubjectStepEndedAll, meshbuilds.SubjectVersionReserved}}); err != nil {
|
||||
// builder's version-reserved, which no stream of its own keeps (design 47 H2 §805), and the Go package registry seat's
|
||||
// published, a series' newest a pin names (H-A-495 addendum).
|
||||
FilterSubjects: []string{"mesh.mod." + assignerHolderModule + ".event.views.>", meshcatalog.SubjectRoom, nodebuildgo.SubjectStepEndedAll, meshbuilds.SubjectVersionReserved,
|
||||
gopackageregistry.SubjectPublished}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
em := views.Emitter{Owner: assignerHolderModule, Subject: process.Account{Node: "n1", Module: assignerHolderModule}.EventSubject,
|
||||
|
||||
@@ -664,7 +664,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
sources.refresh()
|
||||
})
|
||||
// The held take-ins are taken in again from the state held at the start, before a branch's consumer gives a later one
|
||||
// (design 47 H, H-A-114, H-A-118).
|
||||
// (design 47 H, H-A-114, H-A-118); a chain's key a stop left is deleted first (H2 H-A-497 addendum).
|
||||
ix.EndChains(ctx)
|
||||
sources.takeAgain()
|
||||
sources.refresh()
|
||||
if stopped != nil {
|
||||
|
||||
+76
-12
@@ -19,9 +19,11 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -76,10 +78,11 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name
|
||||
// of lib-a and one of lib-b, and so their trees alone, moves lib-a, lib-b and their importers alone, never the other 32
|
||||
// modules whose repository's commit moved too.
|
||||
//
|
||||
// Design 47 H2 §805 (H-A-453 (3)): the builder's twenty version-reserved of mesh-sdk in that revision's pipeline (twenty
|
||||
// artifacts of its Go package, as repin_test.go reads them) are kept in heard and gathered, and its pipeline-started keeps one
|
||||
// re-pin revision of main, importer its mover pinned to the reserved version; the next pipeline's reservation keeps the
|
||||
// revision after it, so none came between.
|
||||
// Design 47 H2 §805 (H-A-453 (3)) and H-A-495 with its addendum: the builder's twenty version-reserved of mesh-sdk in that
|
||||
// revision's pipeline (twenty artifacts of its Go package, as repin_test.go reads them) are kept in heard and gathered, and
|
||||
// they and its pipeline-started move no pin: a pin names only a published version. The release of the reserved version (the
|
||||
// Go package registry seat's published and every artifact's package-published) keeps one re-pin revision of main, importer
|
||||
// its mover pinned to it; the next pipeline's reservation keeps none, so the revision after it is the next release's.
|
||||
func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
const (
|
||||
sdkGo = "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -143,6 +146,24 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
if heardKV, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.HeardState), History: 64, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// package-published on its kept stream with the indexer's durable, as the grants part makes them, and the state versions
|
||||
// a release is recorded in (H-A-495: the release re-pins).
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.VersionsState), History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: meshbuilds.StreamPackagePublished, Subjects: []string{meshbuilds.SubjectPackagePublished}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, meshbuilds.StreamPackagePublished, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
MaxDeliver: EventDeliveries}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The indexer's durable on pipeline-started samples every acknowledgement, so the bed hears that the settle a
|
||||
// pipeline-started makes (H-A-424 (1)) has ended before it releases, never by sleeping.
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, meshpipelinerunner.StreamPipelineStarted, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer",
|
||||
AckPolicy: jetstream.AckExplicitPolicy, FilterSubjects: []string{meshpipelinerunner.SubjectPipelineStartedAll}, SampleFrequency: "100%"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A dry run's key left by an indexer that stopped mid-run (§804).
|
||||
if dryRuns, err = js.KeyValue(ctx, index.Bucket(testIndexer, index.DryRunsState)); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -260,8 +281,9 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
t.Errorf("the catalogue's commit changing lib-a's and lib-b's tests moves %v of its 36 modules, want lib-a, lib-b and their importers alone", movers)
|
||||
}
|
||||
|
||||
// §805: twenty reservations of mesh-sdk's packages in main's revision of source 12, kept and gathered; its pipeline-started
|
||||
// keeps one re-pin revision of main, and the next pipeline's reservation the one after it.
|
||||
// §805, H-A-495: twenty reservations of mesh-sdk's packages in main's revision of source 12, kept and gathered, and its
|
||||
// pipeline-started move no pin; the release of the reserved version keeps one re-pin revision of main, and the next
|
||||
// pipeline's reservation none, so the revision after it is the next release's.
|
||||
objects := newBusObjects(js, testAccount, index.CatalogRevisionsObjects)
|
||||
untilRevision := func(n int64) meshcatalog.CatalogRevision {
|
||||
t.Helper()
|
||||
@@ -299,19 +321,61 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
return v != nil && json.Unmarshal(v, &r) == nil && len(r.Reservations) == artifacts
|
||||
})
|
||||
}
|
||||
// release publishes mesh-sdk's Go package at version, as the registry seat says it (the series' newest a pin names,
|
||||
// H-A-495 addendum), and every artifact of mesh-sdk's entry in main's revision n, as the builder says them.
|
||||
release := func(n int64, version string) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(gopackageregistry.PublishedEvent{Kind: "package", Name: sdkGo, Module: sdkGo, Version: version, Commit: m2, Digest: "sha256:" + version})
|
||||
if _, err := js.Publish(ctx, gopackageregistry.SubjectPublished, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, entries, err := index.ReadRevision(ctx, objects, index.BranchObject("main", n))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
i := slices.IndexFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == "mesh-sdk" })
|
||||
if i < 0 {
|
||||
t.Fatalf("main.%d holds no entry of mesh-sdk", n)
|
||||
}
|
||||
for _, a := range entries[i].Artifacts {
|
||||
raw, _ := json.Marshal(meshbuilds.PackagePublished{Module: "mesh-sdk", Artifact: a.Ref, Digest: "sha256:" + a.Ref + version, Version: version, Source: 12, Branch: "main"})
|
||||
if _, err := js.Publish(ctx, meshbuilds.SubjectPackagePublished, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// started says pipeline-started of main's revision n and waits for the indexer's acknowledgement of it, which follows its settle.
|
||||
started := func(n int64) {
|
||||
t.Helper()
|
||||
acks, err := js.Conn().SubscribeSync("$JS.EVENT.METRIC.CONSUMER.ACK." + meshpipelinerunner.StreamPipelineStarted + ".n1_mesh-delivery-indexer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = acks.Unsubscribe() }()
|
||||
if err := js.Conn().Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sayPipeline(t, ctx, js, false, "main", n)
|
||||
if _, err := acks.NextMsg(waitBound); err != nil {
|
||||
t.Fatalf("the pipeline-started of main.%d is not acknowledged after %s: %v", n, waitBound, err)
|
||||
}
|
||||
}
|
||||
r := taken.Revision
|
||||
reserve(r, "0.1.28", 20)
|
||||
sayPipeline(t, ctx, js, false, "main", r)
|
||||
if rev := untilRevision(r + 1); rev.CallKind != meshcatalog.CatalogRevisionCallKindRelease || !strings.Contains(rev.Call, index.BranchObject("main", r)+"'s") ||
|
||||
started(r)
|
||||
release(r, "0.1.28")
|
||||
if rev := untilRevision(r + 1); rev.CallKind != meshcatalog.CatalogRevisionCallKindRelease || rev.Call != index.ReleaseCall("mesh-sdk", "0.1.28") ||
|
||||
len(rev.Movers) != 1 || rev.Movers[0].Module != "importer" {
|
||||
t.Errorf("main.%d = %+v, want one re-pin revision of the twenty reservations, importer its mover", r+1, rev)
|
||||
t.Errorf("main.%d = %+v, want the re-pin revision of the release of mesh-sdk 0.1.28, importer its mover: twenty reservations move no pin", r+1, rev)
|
||||
}
|
||||
untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Packages[sdkGo] == "v0.1.28" })
|
||||
reserve(r+1, "0.1.29", 1)
|
||||
sayPipeline(t, ctx, js, false, "main", r+1)
|
||||
if rev := untilRevision(r + 2); !strings.Contains(rev.Call, index.BranchObject("main", r+1)+"'s") {
|
||||
t.Errorf("main.%d = %+v, want the next pipeline's re-pin: the twenty reservations kept more than one revision", r+2, rev)
|
||||
started(r + 1)
|
||||
release(r+1, "0.1.29")
|
||||
if rev := untilRevision(r + 2); rev.Call != index.ReleaseCall("mesh-sdk", "0.1.29") {
|
||||
t.Errorf("main.%d = %+v, want the re-pin of the release of mesh-sdk 0.1.29: the reservation of 0.1.29 keeps no revision", r+2, rev)
|
||||
}
|
||||
untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Packages[sdkGo] == "v0.1.29" })
|
||||
|
||||
// A branch's take-in moving mesh-sdk to m3 is a dry run: its packages stay in dry-runs while it runs, the trunk's are
|
||||
// untouched, and its keys go once it answered (§804).
|
||||
|
||||
@@ -265,24 +265,34 @@ type actor struct {
|
||||
s *sourceMoved
|
||||
}
|
||||
|
||||
// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending).
|
||||
// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending), beside the asks' loop, and answers
|
||||
// at once (design 47 H2 H-A-497 and its addendum): a chain that runs already answers its state from the state takeins; held
|
||||
// sources start the chain at the first, written there before it runs, answered with how many wait.
|
||||
func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn {
|
||||
r := index.TakenIn{Module: branch}
|
||||
if hs, err := a.HeldOf(ctx, branch); err != nil || len(hs) == 0 {
|
||||
if c, running, err := a.ChainOf(ctx, branch); err != nil {
|
||||
r.Answer = "the state takeins is not read: " + err.Error()
|
||||
return r
|
||||
} else if running {
|
||||
r.Held = true
|
||||
r.Answer = fmt.Sprintf("a chain runs: source %d taken in since %s, %d wait", c.Source, c.Since.Format(time.RFC3339), c.Waiting)
|
||||
return r
|
||||
}
|
||||
hs, err := a.HeldOf(ctx, branch)
|
||||
if err != nil || len(hs) == 0 {
|
||||
r.Answer = "the branch " + branch + " holds no source-moved"
|
||||
if err != nil {
|
||||
r.Answer = "the state held is not read: " + err.Error()
|
||||
}
|
||||
return r
|
||||
}
|
||||
a.s.takeAgain()
|
||||
hs, err := a.HeldOf(ctx, branch)
|
||||
still := err != nil || len(hs) > 0
|
||||
r.Stored, r.Held = !still, still
|
||||
r.Answer = "taken in"
|
||||
if still {
|
||||
r.Answer = "still held: indexer_pending says why"
|
||||
}
|
||||
c := a.StartChain(ctx, hs)
|
||||
go func() {
|
||||
a.s.takeAgain()
|
||||
a.EndChain(a.s.ctx, branch)
|
||||
}()
|
||||
r.Held = true
|
||||
r.Answer = fmt.Sprintf("chain started at source %d, %d wait", c.Source, c.Waiting)
|
||||
return r
|
||||
}
|
||||
|
||||
|
||||
@@ -97,6 +97,98 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// chainStore is a takeins state in memory, safe beside the chain's own goroutine, which says on ended when a chain key goes.
|
||||
type chainStore struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
ended chan string
|
||||
}
|
||||
|
||||
func (c *chainStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
v, ok := c.vals[k]
|
||||
return v, 1, ok, nil
|
||||
}
|
||||
func (c *chainStore) Keys(context.Context) ([]string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var out []string
|
||||
for k := range c.vals {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (c *chainStore) Create(_ context.Context, k string, v []byte) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.vals[k] = v
|
||||
return nil
|
||||
}
|
||||
func (c *chainStore) Update(ctx context.Context, k string, v []byte, _ uint64) error {
|
||||
return c.Create(ctx, k, v)
|
||||
}
|
||||
func (c *chainStore) Delete(_ context.Context, k string, _ uint64) error {
|
||||
c.mu.Lock()
|
||||
_, had := c.vals[k]
|
||||
delete(c.vals, k)
|
||||
c.mu.Unlock()
|
||||
if had && strings.HasPrefix(k, index.ChainPrefix) {
|
||||
c.ended <- k
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-497 and its addendum: a hand ask of a branch with held sources starts the chain beside the asks' loop
|
||||
// and answers at once; an ask that meets the running chain answers its state, which the state takeins holds and
|
||||
// indexer_pending shows; the chain's key goes at its end.
|
||||
func TestAHandAskOfAHeldBranchAnswersAtOnce(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
held := &memStore{vals: map[string][]byte{}}
|
||||
takeIns := &chainStore{vals: map[string][]byte{}, ended: make(chan string, 4)}
|
||||
ix := &index.Indexer{Held: held, TakeIns: takeIns, Logf: func(string, ...any) {},
|
||||
Now: func() time.Time { return time.Date(2026, 11, 22, 21, 7, 0, 0, time.UTC) }}
|
||||
handlers := map[string]func(jetstream.Msg){}
|
||||
s := &sourceMoved{ctx: ctx, ix: ix,
|
||||
branches: func(context.Context) ([]string, error) { return []string{"main"}, nil },
|
||||
bind: func(_ context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
|
||||
handlers[branch] = each
|
||||
return func() {}, nil
|
||||
},
|
||||
logf: func(string, ...any) {}}
|
||||
s.refresh()
|
||||
for _, n := range []int{114, 115, 116} {
|
||||
m := &fakeMsg{subject: fmt.Sprintf("mesh.seat.mesh-delivery.event.source-moved.main.%d", n),
|
||||
data: []byte(fmt.Sprintf(`{"v":1,"branch":"main","source":%d,"call":"%d123456789abcde","heads":[]}`, n, n))}
|
||||
handlers["main"](msgOf{f: m})
|
||||
}
|
||||
a := actor{ix, s}
|
||||
s.again.Lock() // the chain waits for the lock, so it is still running at the second ask
|
||||
if r := a.HeldAgain(ctx, "main"); r.Answer != "chain started at source 114, 2 wait" {
|
||||
s.again.Unlock()
|
||||
t.Fatalf("the first ask answered %+v, want the chain started at once", r)
|
||||
}
|
||||
r := a.HeldAgain(ctx, "main")
|
||||
var pending any
|
||||
var err error
|
||||
for _, tool := range index.HeldTools(held, takeIns, nil, ix.Now, nil) {
|
||||
if tool.Name == index.PendingTool {
|
||||
pending, err = tool.Run(map[string]any{})
|
||||
}
|
||||
}
|
||||
s.again.Unlock()
|
||||
chains := pending.(index.PendingAnswer).Chains
|
||||
if r.Answer != "a chain runs: source 114 taken in since 2026-11-22T21:07:00Z, 2 wait" || !r.Held {
|
||||
t.Errorf("the ask meeting the chain answered %+v, want its state at once", r)
|
||||
}
|
||||
if err != nil || len(chains) != 1 || chains[0].Branch != "main" || chains[0].Source != 114 || chains[0].Waiting != 2 {
|
||||
t.Errorf("indexer_pending shows chains %+v (%v), want main's at 114 with 2 waiting", chains, err)
|
||||
}
|
||||
if k := <-takeIns.ended; k != index.ChainKey("main") {
|
||||
t.Errorf("the key %s went, want main's chain", k)
|
||||
}
|
||||
}
|
||||
|
||||
// revisionObjects is the catalog revisions' object bucket in memory.
|
||||
type revisionObjects map[string][]byte
|
||||
|
||||
|
||||
@@ -33,8 +33,8 @@ type takerIn interface {
|
||||
// written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
|
||||
// answered already (this indexer's own write) or deleted is left.
|
||||
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) {
|
||||
return // a running take-in's waiting-on is the indexer's own record, no ask (H-A-409)
|
||||
if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) || strings.HasPrefix(e.Key(), index.ChainPrefix) {
|
||||
return // a running take-in's waiting-on and a running chain are the indexer's own records, no ask (H-A-409, H-A-497)
|
||||
}
|
||||
a, err := index.DecodeTakeIn(e.Value())
|
||||
if err != nil {
|
||||
|
||||
@@ -51,6 +51,9 @@ type PendingAnswer struct {
|
||||
Held []HeldTakeIn `json:"held"`
|
||||
// Running are the take-ins that run, each with what it waits on (design 47 H2 §803, H-A-409).
|
||||
Running []RunningTakeIn `json:"running,omitempty"`
|
||||
// Chains are the running chains of held take-ins, each branch's source being taken in, since when and how many wait
|
||||
// (H-A-497 addendum).
|
||||
Chains []HeldChain `json:"chains,omitempty"`
|
||||
}
|
||||
|
||||
// ErrHeldUnwritten says a source-moved that could not be taken in is not written into the state held either: its message
|
||||
@@ -167,26 +170,153 @@ func (ix *Indexer) unhold(ctx context.Context, branch string, source int64) {
|
||||
}
|
||||
}
|
||||
|
||||
// TakeHeldIn takes every branch's held source revisions in, each branch in its order, on the indexer's own room and at its
|
||||
// start (H-A-114, H-A-118): a branch stops at the first that still cannot be taken in, which stays held with those behind it.
|
||||
// TakeHeldIn takes every branch's held source revisions in, each branch in its order, on the indexer's own room, at its
|
||||
// start and by hand (H-A-114, H-A-118): a branch stops at the first that still cannot be taken in, which stays held with those
|
||||
// behind it. Once a held one is taken in, the next held of its branch is taken in at once, each such start said in the journal
|
||||
// (design 47 H2 H-A-497).
|
||||
func (ix *Indexer) TakeHeldIn(ctx context.Context) {
|
||||
all, err := ReadHeld(ctx, ix.Held)
|
||||
if err != nil {
|
||||
ix.logf("take-in: the state %s is not read, so no held take-in is taken in again now: %v", HeldState, err)
|
||||
return
|
||||
}
|
||||
stopped := map[string]bool{}
|
||||
stopped, landed := map[string]bool{}, map[string]int64{}
|
||||
behind := map[string]int{}
|
||||
for _, h := range all {
|
||||
behind[h.Branch]++
|
||||
}
|
||||
for i, h := range all {
|
||||
behind[h.Branch]--
|
||||
if stopped[h.Branch] {
|
||||
continue
|
||||
}
|
||||
if _, err := ix.TakeInSource(ctx, h.Event()); err != nil {
|
||||
if prev, ok := landed[h.Branch]; ok {
|
||||
ix.logf("take-in: held source %d of %s taken in next, after %d (H-A-497)", h.Source, h.Branch, prev)
|
||||
}
|
||||
ix.writeChain(ctx, HeldChain{Branch: h.Branch, Source: h.Source, Since: ix.now().UTC(), Waiting: behind[h.Branch]})
|
||||
if _, err := ix.TakeInSource(ctx, h.Event()); err == nil {
|
||||
landed[h.Branch] = h.Source
|
||||
} else {
|
||||
ix.logf("take-in: source revision %d of %s is still held: %v", h.Source, h.Branch, err)
|
||||
stopped[h.Branch] = true
|
||||
}
|
||||
// The chain of a branch ends at its first held again or its last held (H-A-497 addendum).
|
||||
if stopped[h.Branch] || !slices.ContainsFunc(all[i+1:], func(n HeldTakeIn) bool { return n.Branch == h.Branch }) {
|
||||
ix.endChain(ctx, h.Branch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ChainPrefix is the takeins keys of the chains of held take-ins: `chain.<branch>`, the branch escaped as a key's name is,
|
||||
// holding the branch's chain while it runs (design 47 H2 H-A-497 and its addendum): written at each step, deleted at its end.
|
||||
const ChainPrefix = "chain."
|
||||
|
||||
// ChainKey is the takeins key of branch's chain.
|
||||
func ChainKey(branch string) string { return ChainPrefix + keys.EscapeName(branch) }
|
||||
|
||||
// HeldChain is a running chain of a branch's held take-ins (H-A-497 addendum): the source being taken in, since when, and
|
||||
// how many held sources of the branch wait behind it.
|
||||
type HeldChain struct {
|
||||
Branch string `json:"branch"`
|
||||
Source int64 `json:"source"`
|
||||
Since time.Time `json:"since"`
|
||||
Waiting int `json:"waiting"`
|
||||
}
|
||||
|
||||
// writeChain writes branch's chain at its step c into the state takeins; a failed write is said.
|
||||
func (ix *Indexer) writeChain(ctx context.Context, c HeldChain) {
|
||||
if ix.TakeIns == nil {
|
||||
return
|
||||
}
|
||||
data, err := json.Marshal(c)
|
||||
if err == nil {
|
||||
err = write(ctx, ix.TakeIns, ChainKey(c.Branch), func([]byte, bool) ([]byte, bool, error) { return data, false, nil })
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("take-in: the chain of %s at source revision %d is not written into %s: %v", c.Branch, c.Source, TakeInsState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// EndChain deletes branch's chain from the state takeins once the take-in its ask started returned, so a chain whose held
|
||||
// sources another act took first is never left running.
|
||||
func (ix *Indexer) EndChain(ctx context.Context, branch string) { ix.endChain(ctx, branch) }
|
||||
|
||||
// EndChains deletes every chain the state takeins holds, at the indexer's start (H-A-497 addendum): no chain runs before the
|
||||
// start's take-in, so a key left by a stop between a chain's last take-in and its end would answer "a chain runs" for good.
|
||||
func (ix *Indexer) EndChains(ctx context.Context) {
|
||||
chains, err := ReadChains(ctx, ix.TakeIns)
|
||||
if err != nil {
|
||||
ix.logf("take-in: the chains in %s are not read, so none left by a stop is deleted: %v", TakeInsState, err)
|
||||
return
|
||||
}
|
||||
for _, c := range chains {
|
||||
ix.logf("take-in: the chain of %s at source revision %d, left by a stop, is deleted at the start", c.Branch, c.Source)
|
||||
ix.endChain(ctx, c.Branch)
|
||||
}
|
||||
}
|
||||
|
||||
// endChain deletes branch's chain from the state takeins.
|
||||
func (ix *Indexer) endChain(ctx context.Context, branch string) {
|
||||
if ix.TakeIns == nil {
|
||||
return
|
||||
}
|
||||
if err := write(ctx, ix.TakeIns, ChainKey(branch), func([]byte, bool) ([]byte, bool, error) { return nil, true, nil }); err != nil {
|
||||
ix.logf("take-in: the chain of %s is not deleted from %s: %v", branch, TakeInsState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// StartChain writes branch's chain at its first held source, before the chain is taken in beside the asks (H-A-497
|
||||
// addendum), so an ask after it finds it running; it answers that chain.
|
||||
func (ix *Indexer) StartChain(ctx context.Context, held []HeldTakeIn) HeldChain {
|
||||
c := HeldChain{Branch: held[0].Branch, Source: held[0].Source, Since: ix.now().UTC(), Waiting: len(held) - 1}
|
||||
ix.writeChain(ctx, c)
|
||||
return c
|
||||
}
|
||||
|
||||
// ChainOf is branch's running chain in the state takeins, and whether one runs.
|
||||
func (ix *Indexer) ChainOf(ctx context.Context, branch string) (HeldChain, bool, error) {
|
||||
if ix.TakeIns == nil {
|
||||
return HeldChain{}, false, nil
|
||||
}
|
||||
v, _, found, err := ix.TakeIns.Get(ctx, ChainKey(branch))
|
||||
if err != nil || !found {
|
||||
return HeldChain{}, false, err
|
||||
}
|
||||
var c HeldChain
|
||||
if err := json.Unmarshal(v, &c); err != nil {
|
||||
return HeldChain{}, false, fmt.Errorf("%s.%s: %w", TakeInsState, ChainKey(branch), err)
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
|
||||
// ReadChains are the running chains the state takeins holds (H-A-497 addendum), each branch's once.
|
||||
func ReadChains(ctx context.Context, st Store) ([]HeldChain, error) {
|
||||
if st == nil {
|
||||
return nil, nil
|
||||
}
|
||||
ks, err := st.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(ks)
|
||||
var out []HeldChain
|
||||
for _, k := range ks {
|
||||
if !strings.HasPrefix(k, ChainPrefix) {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := st.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var c HeldChain
|
||||
if !found || json.Unmarshal(v, &c) != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SkipTakeIn ends a held source revision unwritten by hand (indexer_take_in_skip, §744): it says take-in.taken with no entries,
|
||||
// skipped, by and why, and deletes the branch's held take-in; the caller acknowledges the source-moved.
|
||||
func (ix *Indexer) SkipTakeIn(ctx context.Context, e meshdelivery.SourceMovedEvent, by, why string) error {
|
||||
@@ -236,7 +366,7 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs
|
||||
{Name: PendingTool,
|
||||
Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " +
|
||||
"could not take in (the catalog store full, a read of the forge unanswered), why and since when, in order, each kept here once " +
|
||||
"acknowledged and taken in again on the indexer's room and at its start; and each running take-in that waits over a minute on another part, its waiting-on (part, verb, since). indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.",
|
||||
"acknowledged and taken in again on the indexer's room and at its start; each running take-in that waits over a minute on another part, its waiting-on (part, verb, since); and each running chain of held take-ins, the source being taken in, since when and how many wait. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
h, err := ReadHeld(context.Background(), held)
|
||||
@@ -244,11 +374,16 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs
|
||||
return PendingAnswer{Held: h}, err
|
||||
}
|
||||
r, err := ReadRunning(context.Background(), takeIns)
|
||||
return PendingAnswer{Held: h, Running: r}, err
|
||||
if err != nil {
|
||||
return PendingAnswer{Held: h, Running: r}, err
|
||||
}
|
||||
c, err := ReadChains(context.Background(), takeIns)
|
||||
return PendingAnswer{Held: h, Running: r, Chains: c}, err
|
||||
}},
|
||||
{Name: TakeInPendingTool,
|
||||
Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " +
|
||||
"answers whether it was taken in. indexer_pending names the held branches.",
|
||||
"the held sources are taken in in order beside the asks, and it answers at once that the chain started at a source and " +
|
||||
"how many wait, or, while a chain runs, its state (design 47 H2 H-A-497). indexer_pending names the held branches and the running chains.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"branch"}, "properties": map[string]any{
|
||||
"branch": map[string]any{"type": "string", "description": "the watched branch, as indexer_pending names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken again now"}}},
|
||||
|
||||
@@ -73,8 +73,8 @@ func pinObject(key string) (string, bool) {
|
||||
}
|
||||
|
||||
// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a
|
||||
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it and every
|
||||
// other revision the record pins (H-A-147). A body
|
||||
// pipeline-started pins the revision it names, and each revision its movers are kept at (design 47 H2 H-A-496), under the
|
||||
// record `pipeline.<pipeline>`, its pipeline-ended frees it and every other revision the record pins (H-A-147). A body
|
||||
// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it
|
||||
// again; one applied twice changes nothing.
|
||||
func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error {
|
||||
@@ -83,7 +83,15 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
}
|
||||
switch {
|
||||
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
||||
if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)); err != nil {
|
||||
record := PipelineRecord(started.Pipeline)
|
||||
if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), record); err != nil {
|
||||
return err
|
||||
}
|
||||
// Every revision the pipeline's movers are kept at is pinned under its record too, so no judge removes one it still
|
||||
// reads (design 47 H2 H-A-496); the latest started's kept set is the pipeline's kept pins, so a kept revision the
|
||||
// record pins that this started no longer names is freed (H-A-496 addendum). A kept revision is always newer than the
|
||||
// pipeline's own (a re-pin of its own source, H-A-283), so its own revision and its validate base stay pinned.
|
||||
if err := ix.pinKept(ctx, started.Branch, started.Revision, started.Kept, record); err != nil {
|
||||
return err
|
||||
}
|
||||
// The reservations of the pipeline's revision settle at its start, if they gather (design 47 H2 §805, H-A-424 (1)).
|
||||
@@ -229,6 +237,41 @@ func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// pinKept pins each of kept, the revisions a pipeline of branch at revision own keeps movers at, under record, and frees
|
||||
// every revision newer than own the record pins that kept does not name (design 47 H2 H-A-496 and its addendum).
|
||||
func (ix *Indexer) pinKept(ctx context.Context, branch string, own int64, kept []int64, record string) error {
|
||||
for _, r := range kept {
|
||||
if r <= 0 || r == own {
|
||||
continue
|
||||
}
|
||||
if err := ix.pin(ctx, PinKey(branch, r), record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
b, r, ok := ParsePinKey(k)
|
||||
if !ok || b != branch || r <= own || slices.Contains(kept, r) {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pv PinValue
|
||||
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
|
||||
continue
|
||||
}
|
||||
if err := ix.unpin(ctx, k, record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PipelineRecord is the record a pipeline pins its revision under: `pipeline.<branch>.<source>` (design 47 H §743, A-449a).
|
||||
func PipelineRecord(pipeline string) string { return "pipeline." + pipeline }
|
||||
|
||||
|
||||
@@ -13,6 +13,47 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 H2 H-A-496 and its addendum: a pipeline-started pins the pipeline's own revision and each revision its movers
|
||||
// are kept at under the pipeline's record; a later started (said again at a re-pin) is the pipeline's kept set, freeing a
|
||||
// kept revision it no longer names but never the pipeline's own; pipeline-ended frees them all.
|
||||
func TestAPipelinesKeptRevisionsArePinnedUntilItEnds(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pending := uint64(1) // the keeps judge nothing while the consumer is behind
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
|
||||
Now: func() time.Time { return time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC) }, PinsPending: func(context.Context) (uint64, error) { return pending, nil }}
|
||||
kept := func() []string {
|
||||
n, _ := ix.Revisions.Names(ctx)
|
||||
return slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok })
|
||||
}
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||
for source := int64(1); source <= 6; source++ {
|
||||
e := reader
|
||||
e.Commit = fmt.Sprintf("c%d", source)
|
||||
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pending = 0
|
||||
heard := func(started *runner.PipelineStarted, ended *runner.PipelineEnded, want ...string) {
|
||||
t.Helper()
|
||||
if err := ix.HeardPins(ctx, started, ended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
if got := kept(); !slices.Equal(got, want) {
|
||||
t.Errorf("kept %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{2, 3, 4}}, nil,
|
||||
"main.1", "main.2", "main.3", "main.4", "main.6")
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, "main.3"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.1"}) {
|
||||
t.Errorf("the pin of main.3 holds %s, want the pipeline's record", v)
|
||||
}
|
||||
heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{3, 4}}, nil,
|
||||
"main.1", "main.3", "main.4", "main.6")
|
||||
heard(nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.1", Revision: 1, Outcome: runner.PipelineEndedOutcomeFailed}, "main.6")
|
||||
}
|
||||
|
||||
// Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its
|
||||
// pipeline-ended, counts a record once, and removes a revision only while its consumers are caught up, judged against the pins
|
||||
// its own state holds, never a count of pins stated: one two pipelines pin stays until both free it, one neither pinned nor
|
||||
|
||||
@@ -101,9 +101,10 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said)
|
||||
}
|
||||
|
||||
// Design 47 H2 §805, H-A-418, H-A-420, H-A-424, H-A-436: a pipeline reserving twenty packages of mesh-sdk in main's
|
||||
// revision 2 keeps one re-pin revision of main when its gathering settles, at its first step-ended; its release then
|
||||
// keeps none on main, and a pipeline that did not pass withdraws its reservations back to the newest released version.
|
||||
// Design 47 H2 H-A-495 (ending H-A-418 (2), H-A-424 (4)), H-A-436: a pin names only a released version. A pipeline
|
||||
// reserving twenty packages of mesh-sdk in main's revision 2 moves no pin when its gathering settles; the release of
|
||||
// the reserved version re-pins g in one revision; a pipeline that did not pass withdraws its reservations and moves
|
||||
// nothing; and a pin held on a version never released is moved to the newest released at the next re-pin, said.
|
||||
ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) {
|
||||
out := map[string]map[string]string{}
|
||||
for _, d := range dirs {
|
||||
@@ -111,6 +112,8 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
var lines []string
|
||||
ix.Logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) }
|
||||
heard := newMem()
|
||||
ix.Heard = heard
|
||||
revision := BranchObject("main", 2)
|
||||
@@ -121,6 +124,7 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
}
|
||||
return rev.Revision
|
||||
}
|
||||
const sdkGo = "git.novox.be/novox/mesh-sdk/go"
|
||||
for i := range 20 {
|
||||
e := meshbuilds.VersionReservedEvent{Module: "mesh-sdk", Artifact: fmt.Sprintf("mesh-sdk:p%d", i), Path: "mesh-sdk/go", Version: "0.1.28", Revision: revision}
|
||||
if err := ix.HearVersionReserved(ctx, e); err != nil {
|
||||
@@ -130,44 +134,57 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
if _, _, open, _ := heard.Get(ctx, SettlingKey(revision)); !open || newest() != 2 {
|
||||
t.Fatalf("twenty reservations: gathering open %v, main's newest %d, want open and no revision kept before it settles", open, newest())
|
||||
}
|
||||
if err := ix.Settle(ctx, revision); err != nil || newest() != 3 {
|
||||
t.Fatalf("the gathering settled (%v) keeping main's newest %d, want one re-pin revision, 3", err, newest())
|
||||
if err := ix.Settle(ctx, revision); err != nil || newest() != 2 {
|
||||
t.Fatalf("the gathering settled (%v) keeping main's newest %d, want still 2: a reservation moves no pin", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" {
|
||||
t.Errorf("g after the reservations is pinned %v, want v0.1.28", now.Packages)
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.27" {
|
||||
t.Errorf("g after the reservations is pinned %v, want the released v0.1.27", now.Packages)
|
||||
}
|
||||
if err := ix.Settle(ctx, revision); err != nil || newest() != 3 {
|
||||
t.Errorf("a second step-ended settled (%v) keeping main's newest %d, want still 3", err, newest())
|
||||
}
|
||||
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.28", Commit: strings.Repeat("1", 40), Published: true}); err != nil {
|
||||
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: sdkGo, Version: "0.1.28", Commit: strings.Repeat("1", 40), Published: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ix.Repin(ctx, "mesh-sdk", "0.1.28"); err != nil || newest() != 3 {
|
||||
t.Errorf("the release of the reserved 0.1.28 re-pinned (%v) keeping main's newest %d, want no revision more", err, newest())
|
||||
t.Errorf("the release of the reserved 0.1.28 re-pinned (%v) keeping main's newest %d, want one revision more, 3", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.28" {
|
||||
t.Errorf("g after the release is pinned %v, want v0.1.28", now.Packages)
|
||||
}
|
||||
next := BranchObject("main", 3)
|
||||
if err := ix.HearVersionReserved(ctx, meshbuilds.VersionReservedEvent{Module: "mesh-sdk", Artifact: "mesh-sdk:p0", Path: "mesh-sdk/go", Version: "0.1.29", Revision: next}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Settle(ctx, next); err != nil || newest() != 4 {
|
||||
t.Fatalf("the reservation of 0.1.29 settled (%v) keeping main's newest %d, want 4", err, newest())
|
||||
if err := ix.Settle(ctx, next); err != nil || newest() != 3 {
|
||||
t.Fatalf("the reservation of 0.1.29 settled (%v) keeping main's newest %d, want still 3", err, newest())
|
||||
}
|
||||
// H-A-479 second addendum: a pipeline at the same revision whose movers are others withdraws none of mesh-sdk's.
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"app"}); err != nil || newest() != 4 {
|
||||
t.Fatalf("another pipeline's withdrawal (%v) kept main's newest %d, want 4: nothing of mesh-sdk withdrawn", err, newest())
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"app"}); err != nil || newest() != 3 {
|
||||
t.Fatalf("another pipeline's withdrawal (%v) kept main's newest %d, want 3: nothing of mesh-sdk withdrawn", err, newest())
|
||||
}
|
||||
if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); !kept {
|
||||
t.Fatalf("another pipeline's withdrawal took mesh-sdk's reservation")
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"mesh-sdk"}); err != nil || newest() != 5 {
|
||||
t.Fatalf("the withdrawal (%v) kept main's newest %d, want 5", err, newest())
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"mesh-sdk"}); err != nil || newest() != 3 {
|
||||
t.Fatalf("the withdrawal (%v) kept main's newest %d, want 3: the withdrawn reservation pinned nothing", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" {
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.28" {
|
||||
t.Errorf("g after the withdrawal is pinned %v, want the newest released v0.1.28", now.Packages)
|
||||
}
|
||||
if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); kept {
|
||||
t.Errorf("the withdrawn reservation is still kept")
|
||||
}
|
||||
stale, _ := moduleOf(t, mods, "g")
|
||||
stale.Packages = map[string]string{sdkGo: "v0.1.29"} // pinned before H-A-495 to the withdrawn reservation
|
||||
mods.put("g", mustEncode(t, stale))
|
||||
lines = nil
|
||||
if _, err := ix.Repin(ctx, "mesh-sdk", "0.1.28"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.28" {
|
||||
t.Errorf("g pinned to the never released v0.1.29 is pinned %v after a re-pin, want v0.1.28", now.Packages)
|
||||
}
|
||||
if said := "g: pinned to " + sdkGo + " v0.1.29, never released; moved to v0.1.28"; !slices.Contains(lines, said) {
|
||||
t.Errorf("the re-pin said %q, want the line %q", lines, said)
|
||||
}
|
||||
// H-A-479 second addendum: two pipelines' movers reserve at one revision while its gathering is open; withdrawing one
|
||||
// pipeline's mover keeps the gathering and the other's reservation, withdrawing the last removes the gathering too.
|
||||
six := BranchObject("main", 6)
|
||||
|
||||
@@ -17,13 +17,13 @@ import (
|
||||
|
||||
// A release re-pins once, at its reservation (design 47 H2 §805; H-A-418, H-A-420, H-A-424). The builder says
|
||||
// version-reserved when it reserves a package's number, naming the revision `<branch>.<n>` its pipeline runs. The indexer
|
||||
// writes the series' numbered version with the contents read at the commit of the module's entry in that revision, so
|
||||
// acceptedSeries and releasedPins pin it, and keeps the reservation in its state heard under
|
||||
// writes the series' numbered version with the contents read at the commit of the module's entry in that revision, and
|
||||
// keeps the reservation in its state heard under
|
||||
// `reserved.<revision>.<module>`, each key living reserved-keep. It gathers a pipeline's reservations per revision and
|
||||
// writes one re-pin revision of that revision's branch when the gathering settles: at the first pipeline-started or
|
||||
// step-ended of that pipeline heard after them, or at the expiry of the key settling.<revision>, written reserved-settle
|
||||
// after the first. The release's package-published then
|
||||
// moves no fingerprint. A pipeline-ended not passed whose revision holds reservations withdraws them: the series'
|
||||
// after the first. A pin names only a released version, never a reservation (design 47 H2 H-A-495), so that re-pin
|
||||
// moves no pin, and the release's re-pin (H-A-165) moves the importers. A pipeline-ended not passed whose revision holds reservations withdraws them: the series'
|
||||
// numbered goes back to its newest released version and the importers are re-pinned to it.
|
||||
|
||||
// ReservedPrefix begins every key of the state heard holding a revision's reservations of one module.
|
||||
|
||||
@@ -198,14 +198,10 @@ func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repos
|
||||
return out, pins
|
||||
}
|
||||
|
||||
// pinnedVersion is the version of a series its users are pinned to: its numbered one when a reservation wrote it with its
|
||||
// contents and it is newer than the newest released (design 47 H2 §805, H-A-418 (2), H-A-424 (4)), else the newest; nil
|
||||
// for none.
|
||||
// pinnedVersion is the version of a series its users are pinned to: its newest released one, never a reservation's
|
||||
// numbered one, which a pipeline that does not pass withdraws unpublished (design 47 H2 H-A-495, which ends H-A-418 (2)
|
||||
// and H-A-424 (4)); nil for none.
|
||||
func pinnedVersion(s SeriesPackage) *SeriesVersion {
|
||||
if n := s.Numbered; n != nil && n.Contents != nil && (s.Newest == nil || CompareVersions("v"+s.Newest.Version, "v"+n.Version) < 0) {
|
||||
p := publishedOfNumbered(*n.Contents)
|
||||
return &SeriesVersion{Version: n.Version, Commit: n.Commit, Digest: n.Digest, Fingerprint: n.Fingerprint, Contents: contentsOfPublished(p)}
|
||||
}
|
||||
return s.Newest
|
||||
}
|
||||
|
||||
|
||||
@@ -371,11 +371,35 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
|
||||
len(held[0].Heads) != 1 || held[0].Heads[0] != moved.Heads[0] || !reflect.DeepEqual(held[0].Event(), moved) {
|
||||
t.Fatalf("held = %+v, want 3 then 4, each with its event whole", held)
|
||||
}
|
||||
// H-A-497: behind them a source whose head the forge does not answer, and one more behind it.
|
||||
for _, n := range []struct {
|
||||
source int64
|
||||
commit string
|
||||
}{{5, "c-unread"}, {6, f.Merge}} {
|
||||
e := moved
|
||||
e.Source, e.Call, e.Heads = n.source, fmt.Sprintf("%d123456789abcdef", n.source), []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: n.commit}}
|
||||
if behind, err := ix.HoldBehind(ctx, e); err != nil || !behind {
|
||||
t.Fatalf("source %d held behind: %v (%v)", n.source, behind, err)
|
||||
}
|
||||
}
|
||||
var lines []string
|
||||
ix.Logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) }
|
||||
ix.TakeIns = newMem()
|
||||
// H-A-497 addendum: while the chain runs, its key names the source being taken in and how many wait behind it.
|
||||
var steps []string
|
||||
read := ix.Read
|
||||
ix.Read = func(ctx context.Context, repository, sha, dir string) (RepoRead, error) {
|
||||
if c, ok, _ := ix.ChainOf(ctx, "main"); ok {
|
||||
steps = append(steps, fmt.Sprintf("%d/%d", c.Source, c.Waiting))
|
||||
}
|
||||
return read(ctx, repository, sha, dir)
|
||||
}
|
||||
objects.capacity = 1 << 22
|
||||
if ok, err := ix.RoomAgain(ctx); err != nil || !ok || rooms != 1 {
|
||||
t.Fatalf("room said %v %d times (%v), want once the store has room", ok, rooms, err)
|
||||
}
|
||||
// H-A-114: on room (and at start) the held ones are taken in again from the state held alone, in their order.
|
||||
// H-A-114, H-A-497: on room (and at start, and by hand) the held ones are taken in again from the state held alone, in
|
||||
// their order, each next one at once and said in the journal, until one is held again: those behind it stay held.
|
||||
ix.TakeHeldIn(ctx)
|
||||
var taken []int64
|
||||
for _, e := range *said {
|
||||
@@ -386,8 +410,38 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
|
||||
if !reflect.DeepEqual(taken, []int64{3, 4}) {
|
||||
t.Errorf("taken again on room: %v of %+v, want 3 then 4", taken, *said)
|
||||
}
|
||||
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 {
|
||||
t.Errorf("held = %+v after the take-in, want none", held)
|
||||
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 2 || held[0].Source != 5 || held[1].Source != 6 {
|
||||
t.Errorf("held = %+v after the take-in, want 5 held again and 6 behind it", held)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"take-in: held source 4 of main taken in next, after 3 (H-A-497)",
|
||||
"take-in: held source 5 of main taken in next, after 4 (H-A-497)",
|
||||
} {
|
||||
if !slices.Contains(lines, want) {
|
||||
t.Errorf("the journal holds no line %q: %q", want, lines)
|
||||
}
|
||||
}
|
||||
for _, l := range lines {
|
||||
if strings.Contains(l, "held source 6 of main taken in") {
|
||||
t.Errorf("source 6 was taken in behind a held one: %q", l)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(steps, "4/2") || !slices.Contains(steps, "5/1") {
|
||||
t.Errorf("the chain's key read at each take-in %v, want main at 4 with 2 waiting and at 5 with 1", steps)
|
||||
}
|
||||
// H-A-497 addendum: the chain's key in the state takeins goes at its end, here its first held again, and one a stop
|
||||
// left is deleted at the start.
|
||||
if chains, err := ReadChains(ctx, ix.TakeIns); err != nil || len(chains) != 0 {
|
||||
t.Errorf("chains %+v (%v) after the chain stopped, want none", chains, err)
|
||||
}
|
||||
ix.writeChain(ctx, HeldChain{Branch: "main", Source: 9, Waiting: 1})
|
||||
if ix.EndChains(ctx); func() bool { c, _ := ReadChains(ctx, ix.TakeIns); return len(c) != 0 }() {
|
||||
t.Error("a chain's key left by a stop stays after the start")
|
||||
}
|
||||
ix.Read = read
|
||||
ix.Logf, ix.TakeIns = nil, nil
|
||||
for _, src := range []int64{5, 6} {
|
||||
ix.unhold(ctx, "main", src)
|
||||
}
|
||||
// A skip ends a held source revision unwritten: take-in.taken with no entries, skipped, by and why; held is deleted.
|
||||
ix.hold(ctx, meshdelivery.SourceMovedEvent{Branch: "nox-mesh", Source: 9, Call: "fedcba9876543210"}, "the catalog store is full")
|
||||
|
||||
@@ -236,6 +236,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
fp, none string
|
||||
tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint
|
||||
pkgs sdkstep.PackageVersions // the package versions, when there is a fingerprint
|
||||
moved []string // the pins moved off a version never released, said (H-A-495)
|
||||
}
|
||||
done := map[string]result{}
|
||||
visiting := map[string]bool{}
|
||||
@@ -313,6 +314,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
// a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
|
||||
r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository, kept.Packages)
|
||||
r.tcs = toolchainsUsed(toolchains)
|
||||
r.moved = neverReleased(module, k.held.Packages, r.pkgs, series, k.manifest)
|
||||
}
|
||||
}
|
||||
if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
|
||||
@@ -325,6 +327,9 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
}
|
||||
done[module] = r
|
||||
k.cur.Inputs2 = &kept
|
||||
for _, line := range r.moved {
|
||||
ix.logf("%s", line)
|
||||
}
|
||||
return r
|
||||
}
|
||||
finals := map[string]meshcatalog.ModulesByModuleValue{}
|
||||
|
||||
@@ -140,7 +140,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
|
||||
}
|
||||
var want []string
|
||||
for _, a := range entry.Artifacts {
|
||||
want = append(want, a.Ref)
|
||||
if a.Kind != catalogue.ArtifactUpstream { // an upstream copy is released by its pinned digest, never published (H-A-486)
|
||||
want = append(want, a.Ref)
|
||||
}
|
||||
}
|
||||
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
|
||||
var have published
|
||||
@@ -211,7 +213,15 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
|
||||
at := ix.now().UTC()
|
||||
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
|
||||
pins, err := upstreamPins(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if a.Kind == catalogue.ArtifactUpstream {
|
||||
v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: a.Ref, Digest: pins[a.Ref]})
|
||||
continue
|
||||
}
|
||||
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
|
||||
}
|
||||
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
|
||||
@@ -255,6 +265,46 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
return ix.forgetPublished(ctx, pendingKey)
|
||||
}
|
||||
|
||||
// upstreamPins are the pinned digests of the entry's upstream artifacts, by reference, read from its manifest's from
|
||||
// (`<image>@sha256:<digest>`): design 51 copies an upstream image by that digest, so a release names it (H-A-486). An upstream
|
||||
// artifact whose manifest names no pin refuses the release, naming it.
|
||||
func upstreamPins(entry meshcatalog.CatalogEntry) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
if !slices.ContainsFunc(entry.Artifacts, func(a meshcatalog.Artifact) bool { return a.Kind == catalogue.ArtifactUpstream }) {
|
||||
return out, nil
|
||||
}
|
||||
raw, err := json.Marshal(entry.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
From string `json:"from"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s's entry does not read, so its upstream artifacts' pins are not known: %w", entry.Module, err)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactUpstream {
|
||||
continue
|
||||
}
|
||||
if _, d, ok := strings.Cut(a.From, "@"); ok && strings.HasPrefix(d, "sha256:") {
|
||||
out[ArtifactRefOf(entry.Module, a.Name)] = d
|
||||
}
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if _, ok := out[a.Ref]; a.Kind == catalogue.ArtifactUpstream && !ok {
|
||||
return nil, fmt.Errorf("%s is an upstream artifact whose manifest names no pinned digest (from <image>@sha256:<digest>), so %s releases no version", a.Ref, entry.Module)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
|
||||
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
|
||||
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
|
||||
@@ -423,6 +473,29 @@ func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, rep
|
||||
return fingerprint.WithPins(f.fp, pins), pins
|
||||
}
|
||||
|
||||
// neverReleased are the lines said of a module whose held pins (was) name a version above its series' newest released one,
|
||||
// a reservation withdrawn unpublished, and whose new pins move it (design 47 H2 H-A-495): "<module>: pinned to <package>
|
||||
// <version>, never released; moved to <version>". nil for none.
|
||||
func neverReleased(module string, was map[string]string, pins sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, p := range slices.Sorted(maps.Keys(was)) {
|
||||
old, now := strings.TrimPrefix(was[p], "v"), pins[p]
|
||||
if _, ok := versionParts(old); !ok || was[p] == now {
|
||||
continue
|
||||
}
|
||||
for _, s := range series {
|
||||
if s.GoModule != p || s.Newest == nil || s.Series != m.AcceptedSeries(s.Name) {
|
||||
continue
|
||||
}
|
||||
if in, _ := SeriesOf(old); in == s.Series && compareVersions(old, strings.TrimPrefix(s.Newest.Version, "v")) > 0 {
|
||||
out = append(out, fmt.Sprintf("%s: pinned to %s %s, never released; moved to %s", module, p, was[p], cmp.Or(now, "no version")))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moduleOfKey is the module a modules key names; "" for none.
|
||||
func moduleOfKey(key string) string {
|
||||
if key == "" {
|
||||
|
||||
@@ -83,6 +83,36 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
|
||||
if compareVersions(oldest, "0.1.5") < 0 {
|
||||
t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest)
|
||||
}
|
||||
|
||||
// H-A-486: an upstream artifact is copied by its pinned digest and never published, so the version is released once every
|
||||
// other artifact is, the upstream one named with the pin its manifest gives; one whose manifest names no pin releases none.
|
||||
pin := "sha256:" + strings.Repeat("ab", 32)
|
||||
nats := func(from string) map[string]meshcatalog.CatalogEntry {
|
||||
return map[string]meshcatalog.CatalogEntry{"nats": {Module: "nats", Manifest: map[string]any{"build": map[string]any{"artifacts": []any{
|
||||
map[string]any{"name": "server", "kind": "upstream", "from": from},
|
||||
map[string]any{"name": "tools", "kind": "bundle", "from": "."}}}},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "nats:server", Kind: "upstream"}, {Ref: "nats:tools", Kind: "bundle", Language: "go"}}}}
|
||||
}
|
||||
for i, c := range []struct {
|
||||
from, version string
|
||||
released bool
|
||||
}{{"nats:2.12-alpine@" + pin, "0.1.1", true}, {"nats:2.12-alpine", "0.1.2", false}} {
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: int64(4 + i), Kind: "trunk"}, nats(c.from))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "nats", Artifact: "nats:tools", Digest: "sha256:tools", Version: c.version, Source: 9, Branch: "main"})
|
||||
got, _ := ModuleVersions(ctx, ix.Versions, "nats")
|
||||
switch {
|
||||
case c.released && (err != nil || len(got) != 1 || len(got[0].Artifacts) != 2 || got[0].Artifacts[0] != (meshcatalog.VersionArtifact{Artifact: "nats:server", Digest: pin})):
|
||||
t.Fatalf("pinned %q: released %+v (%v); want the server named with its pin", c.from, got, err)
|
||||
case !c.released && (err == nil || !strings.Contains(err.Error(), "no pinned digest") || len(got) != 1):
|
||||
t.Fatalf("unpinned %q: released %+v (%v); want a refusal naming the pin", c.from, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or
|
||||
|
||||
@@ -60,3 +60,37 @@ func TestAgedHub(t *testing.T) {
|
||||
t.Errorf("a message asked again is noted applied: %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheTakeInsSwitch: until the takeins durable is bound, the main durable works every take-in; from the bind, it works
|
||||
// only those said at or before the sequence the durable delivers after, and acknowledges the rest with no work
|
||||
// (design 47 H2 H-A-500 fourth addendum).
|
||||
func TestTheTakeInsSwitch(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
boundFrom uint64
|
||||
bound bool
|
||||
seq uint64
|
||||
delivers bool
|
||||
}{
|
||||
{"not bound, any sequence is worked", 0, false, 7, false},
|
||||
{"bound after 5, the take-in at 5 is worked", 5, true, 5, false},
|
||||
{"bound after 5, the take-in at 6 is the durable's", 5, true, 6, true},
|
||||
{"bound on an empty stream, the first take-in is the durable's", 0, true, 1, true},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
sw := &takeInsSwitch{}
|
||||
if row.bound {
|
||||
sw.boundFrom(row.boundFrom)
|
||||
}
|
||||
hub := &oneHub{}
|
||||
worked := false
|
||||
if err := dropTakenIn(hub, sw, func(jetstream.Msg) { worked = true }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub.handle(ackedMsg{seq: row.seq})
|
||||
if worked == row.delivers {
|
||||
t.Errorf("worked by the main durable: %v, want %v", worked, !row.delivers)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,8 +237,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
advancer.Store(&advance)
|
||||
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
|
||||
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327), and binds the
|
||||
// takeins durable when its bind failed at the start (H-A-500 fifth addendum).
|
||||
var bindTakeInsAgain atomic.Pointer[func()]
|
||||
register := func(reg catalog.Registration) {
|
||||
if f := bindTakeInsAgain.Load(); f != nil {
|
||||
(*f)()
|
||||
}
|
||||
if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err)
|
||||
}
|
||||
@@ -321,6 +326,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
if err := hearStepProgress(heard, func(onSeat, ask string) error {
|
||||
if hearer.LateRunAsk("step-progress", ask) {
|
||||
return nil // an ended run's ask: acknowledged at once, nothing read but its ask key and run (H-A-500 (a))
|
||||
}
|
||||
progressed(clock, onSeat, ask, logger.Printf) // the ask's bound is armed again from now (H-A-395)
|
||||
return hearer.RunProgress(ask)
|
||||
}, logger.Printf); err != nil {
|
||||
@@ -333,6 +341,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
taken := func(ask, node string, at time.Time) (*time.Time, error) {
|
||||
if hearer.LateRunAsk("ask-taken", ask) {
|
||||
return nil, nil // an ended run's ask: acknowledged at once, nothing read but its ask key and run (H-A-500 (a))
|
||||
}
|
||||
takenAt, err := hearer.Taken(ask, node, at)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -370,7 +381,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := hearRecovery(heard, kvRecoveries{ctx, recovered}, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearTakenIn(heard, starter, rv.Catalog, logger.Printf); err != nil {
|
||||
// take-in.taken is acted on on the runner's own takeins durable once that is bound, so no step traffic holds a take-in;
|
||||
// until then the main durable works it, and from the bind acknowledges it at once (design 47 H2 H-A-500 (b) and its
|
||||
// addenda, the fourth: no take-in said while the runner was down is lost).
|
||||
takeInsBound := &takeInsSwitch{}
|
||||
if err := dropTakenIn(heard, takeInsBound, takenIn(starter, rv.Catalog, logger.Printf)); err != nil {
|
||||
return err
|
||||
}
|
||||
takeIns := &oneHandler{}
|
||||
takeInsName := cred.EventConsumerName() + "_" + takeInsSuffix
|
||||
takeInsHeard := agedHub{hub: takeIns, ledger: process.NewConsumerLedger(process.EventsStream, takeInsName, consumersSink(consumers), nil, logger.Printf)}
|
||||
if err := hearTakenIn(takeInsHeard, starter, rv.Catalog, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearReleased(heard, hearer, starter.Registration, logger.Printf); err != nil {
|
||||
@@ -387,6 +408,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// their callers against (A-186): registered before the hub starts, read again once its events are live.
|
||||
seats := seatlist.NewPart("runner")
|
||||
seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf)
|
||||
again := bindTakeIns(ctx, js, cred, takeIns.handle, takeInsBound, logger.Printf)
|
||||
bindTakeInsAgain.Store(&again)
|
||||
if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil {
|
||||
logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err)
|
||||
}
|
||||
|
||||
@@ -172,6 +172,7 @@ func testTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
return ""
|
||||
}, logs)
|
||||
}()
|
||||
takeInsHeard(t, ctx, logs)
|
||||
return func() { stop(); <-done }
|
||||
}
|
||||
stopRunner := start()
|
||||
@@ -267,13 +268,16 @@ func testTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
waitFor(t, ctx, logs, func() bool { return get(key).Outcome == want })
|
||||
}
|
||||
|
||||
// Two take-ins in a row: the first runs, its build asked; the second is written waiting under sequence 1 with no step,
|
||||
// says pipeline-started and wait-held.
|
||||
// Two take-ins in a row: the first runs at sequence 1 (every pipeline has one, H-A-504 addendum), its build asked; the
|
||||
// second is written waiting under sequence 2 with no step, says pipeline-started and wait-held.
|
||||
take(3, "app")
|
||||
outcome("main.3", seat.PipelineOutcomeRunning)
|
||||
if p := get("main.3"); p.Seq != 1 {
|
||||
t.Fatalf("main.3 placed at once has seq %d, want 1", p.Seq)
|
||||
}
|
||||
take(4, "lib")
|
||||
if p := get("main.4"); p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != 1 || len(p.Steps) != 0 {
|
||||
t.Fatalf("main.4 = %s seq %d with %d steps, want waiting at 1 with none", p.Outcome, p.Seq, len(p.Steps))
|
||||
if p := get("main.4"); p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != 2 || len(p.Steps) != 0 {
|
||||
t.Fatalf("main.4 = %s seq %d with %d steps, want waiting at 2 with none", p.Outcome, p.Seq, len(p.Steps))
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool { return wasSaid("wait-held.main.4") && wasSaid("pipeline-started.main.4") })
|
||||
if r := running(); !slices.Equal(r, []string{"main.3"}) {
|
||||
|
||||
@@ -52,6 +52,9 @@ func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return
|
||||
// bus proved; the pipeline then ends passed.
|
||||
func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) {
|
||||
t.Run("the queue runs pipelines in turn", testTheQueueRunsPipelinesInTurn)
|
||||
t.Run("a take-in is heard while late events queue on the main durable", testATakeInIsHeardWhileLateEventsQueue)
|
||||
t.Run("a take-in said before the bind is still worked", testATakeInSaidBeforeTheBindIsStillWorked)
|
||||
t.Run("a takeins bind that failed is bound again on registration-changed", testATakeInsBindThatFailedIsBoundAgainOnRegistrationChanged)
|
||||
t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
@@ -384,6 +387,7 @@ func startWhole(t *testing.T, ctx context.Context, regs []catalog.Registration,
|
||||
}, logs)
|
||||
}()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
takeInsHeard(t, ctx, logs)
|
||||
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -500,6 +504,14 @@ func findingsStream(t *testing.T, ctx context.Context, js jetstream.JetStream, m
|
||||
}
|
||||
}
|
||||
|
||||
// takeInsHeard waits until the runner's takeins durable is bound, which the runner makes at its first start from the
|
||||
// moment it is made (H-A-500 (b)); a take-in said before is worked by the main durable (the fourth addendum), but the tests
|
||||
// publish after the bind so that each take-in is heard where the running runner hears it.
|
||||
func takeInsHeard(t *testing.T, ctx context.Context, logs *syncBuffer) {
|
||||
t.Helper()
|
||||
waitFor(t, ctx, logs, func() bool { return strings.Contains(logs.String(), "take-ins: heard on the consumer") })
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bool) {
|
||||
t.Helper()
|
||||
for ctx.Err() == nil {
|
||||
|
||||
@@ -3,12 +3,19 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
@@ -57,10 +64,140 @@ func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), log
|
||||
}
|
||||
}
|
||||
|
||||
// takeInsSuffix names the runner's own second durable on the shared events stream, `<node>_<module>_takeins`, on which alone
|
||||
// take-in.taken is acted on, so a pipeline starts within seconds of its take-in whatever the step traffic on the runner's
|
||||
// main durable (design 47 H2 H-A-500 (b) and its addenda).
|
||||
const takeInsSuffix = "takeins"
|
||||
|
||||
// The ack wait and the deliveries of the takeins durable, as the grants part gives a module's consumers (H-A-500 (b)).
|
||||
const (
|
||||
takeInsAckWait = 30 * time.Second
|
||||
takeInsMaxDeliver = 5
|
||||
)
|
||||
|
||||
// takeInsConsumerConfig is the takeins durable as the runner makes it: take-in.taken alone, acknowledged one by one, from
|
||||
// the moment it is made (H-A-500 (b)).
|
||||
func takeInsConsumerConfig(cred process.Account) jetstream.ConsumerConfig {
|
||||
cfg := cred.SeatEventConsumerConfig([]string{pipeline.TakenInSubject}, false)
|
||||
cfg.Durable = cred.EventConsumerName() + "_" + takeInsSuffix
|
||||
cfg.AckWait, cfg.MaxDeliver = takeInsAckWait, takeInsMaxDeliver
|
||||
cfg.DeliverPolicy = jetstream.DeliverNewPolicy
|
||||
return cfg
|
||||
}
|
||||
|
||||
// oneHandler is the handlers of one durable that hears one pattern: the takeins durable's (H-A-500 (b)).
|
||||
type oneHandler struct {
|
||||
pattern string
|
||||
handle func(jetstream.Msg)
|
||||
}
|
||||
|
||||
func (o *oneHandler) Handle(pattern string, handle func(jetstream.Msg)) error {
|
||||
if o.handle != nil {
|
||||
return fmt.Errorf("the durable hears %q already, so %q has no taker", o.pattern, pattern)
|
||||
}
|
||||
o.pattern, o.handle = pattern, handle
|
||||
return nil
|
||||
}
|
||||
|
||||
// takeInsSwitch says which durable a take-in.taken is worked on (H-A-500 fourth addendum): until the runner's takeins
|
||||
// durable is bound, the main durable does the work, so a take-in said while the runner was down is lost to no one; from
|
||||
// the bind, the takeins durable alone works the take-ins said after it was made, and the main durable acknowledges those
|
||||
// at once with no work. The boundary is the stream sequence the takeins durable delivers from, so a take-in said before
|
||||
// the durable was made and still waiting on the main durable is worked there whatever the moment it is heard.
|
||||
type takeInsSwitch struct {
|
||||
// from holds 1 + the stream sequence the takeins durable delivers after; 0 while it is not bound.
|
||||
from atomic.Uint64
|
||||
}
|
||||
|
||||
// boundFrom says the takeins durable is bound and delivers every take-in said after stream sequence seq.
|
||||
func (s *takeInsSwitch) boundFrom(seq uint64) { s.from.Store(seq + 1) }
|
||||
|
||||
// delivers says whether the takeins durable delivers m, so the main durable has no work on it.
|
||||
func (s *takeInsSwitch) delivers(m jetstream.Msg) bool {
|
||||
from := s.from.Load()
|
||||
if from == 0 {
|
||||
return false
|
||||
}
|
||||
meta, err := m.Metadata()
|
||||
if err != nil {
|
||||
// A message whose sequence is unknown is worked rather than dropped: the work is done once per key, never twice.
|
||||
return false
|
||||
}
|
||||
return meta.Sequence.Stream >= from
|
||||
}
|
||||
|
||||
// dropTakenIn has the main durable work each take-in.taken the takeins durable does not deliver, and acknowledge the rest at
|
||||
// once with no work, so the finding event-unhandled stays clear (H-A-500 third and fourth addenda).
|
||||
func dropTakenIn(hub handlers, sw *takeInsSwitch, work func(jetstream.Msg)) error {
|
||||
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) {
|
||||
if sw.delivers(m) {
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
work(m)
|
||||
})
|
||||
}
|
||||
|
||||
// bindTakeIns binds the runner's takeins durable and hears it with handle, the durable made when the bus does not hold it
|
||||
// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed. A bind that fails
|
||||
// (the bus has not made the runner's grant yet) is said, and the answered func binds again, called on each
|
||||
// registration-changed as the indexer binds a branch's consumer on each (H-A-500 fifth addendum: nothing waits on a timer
|
||||
// for another part); once bound it does nothing. Once bound, sw says so with the stream sequence the durable delivers
|
||||
// after, and the main durable stops working the take-ins it delivers (the fourth addendum).
|
||||
func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), sw *takeInsSwitch,
|
||||
logf func(string, ...any)) (again func()) {
|
||||
bind := func() error {
|
||||
cons, err := cred.NamedEventConsumer(ctx, js, process.EventsStream, takeInsSuffix)
|
||||
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
cfg := takeInsConsumerConfig(cred)
|
||||
if cons, err = js.CreateConsumer(ctx, process.EventsStream, cfg); err != nil {
|
||||
return fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, process.EventsStream, err)
|
||||
}
|
||||
logf("take-ins: the consumer %s on %s was not there: the runner made it (H-A-500 (b))", cfg.Durable, process.EventsStream)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A durable just made delivers after the stream's last sequence; one found, after what it delivered already.
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s is not read: %w", cred.EventConsumerName(), takeInsSuffix, err)
|
||||
}
|
||||
if _, err = process.HearEvents(cons, handle, logf); err != nil {
|
||||
return err
|
||||
}
|
||||
sw.boundFrom(info.Delivered.Stream)
|
||||
return nil
|
||||
}
|
||||
var mu sync.Mutex
|
||||
bound := false
|
||||
try := func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if bound || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err := bind(); err != nil {
|
||||
logf("take-ins: the consumer %s_%s is not bound, so the main durable works each take-in; bound again at the next registration-changed: %v",
|
||||
cred.EventConsumerName(), takeInsSuffix, err)
|
||||
return
|
||||
}
|
||||
bound = true
|
||||
logf("take-ins: heard on the consumer %s_%s (H-A-500 (b))", cred.EventConsumerName(), takeInsSuffix)
|
||||
}
|
||||
try()
|
||||
return try
|
||||
}
|
||||
|
||||
// hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged
|
||||
// once it is written, and heard again when a write failed, so the key already written starts nothing twice.
|
||||
func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
|
||||
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) {
|
||||
return hub.Handle(pipeline.TakenInSubject, takenIn(starter, v, logf))
|
||||
}
|
||||
|
||||
// takenIn is the work of one take-in.taken, on whichever durable delivers it (H-A-500 fourth addendum).
|
||||
func takenIn(starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) func(jetstream.Msg) {
|
||||
return func(m jetstream.Msg) {
|
||||
// An event of the old form is refused by the contract's strict decode, said in one journal line and ended (§765, H-A-98).
|
||||
ev, err := pipeline.DecodeTakenIn(m.Data())
|
||||
if err != nil {
|
||||
@@ -76,7 +213,7 @@ func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, log
|
||||
}
|
||||
logf("the take-in of %s source %d started %d pipelines", ev.Branch, ev.Source, n)
|
||||
_ = m.Ack()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// hearReleased writes each version-released's version into its mover (design 47 H §746, H-A-52), so the pipeline's end names
|
||||
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// Design 47 H2 H-A-500 (b) and its addenda: a take-in.taken said while 500 late step-progress events wait on the runner's
|
||||
// main durable, its handler held, is heard at once on the runner's own takeins durable, which the runner makes itself; the
|
||||
// main durable acknowledges the take-in with no work. testATakeInIsHeardWhileLateEventsQueue is a row of
|
||||
// TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview (the module's tests are folded under its budget).
|
||||
func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"}
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: cred.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const progress = "mesh.seat.node-build-go.event.step-progress.n1"
|
||||
for i := 0; i < 500; i++ {
|
||||
if _, err := js.Publish(ctx, progress, []byte(`{"ask":"late","phase":"run"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The main durable's step-progress handler is held until the test ends: every late event waits behind the first.
|
||||
held := make(chan struct{})
|
||||
defer close(held)
|
||||
hub := process.NewEventHub(t.Logf)
|
||||
if err := hub.Handle(progress, func(m jetstream.Msg) { <-held; _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sw := &takeInsSwitch{}
|
||||
if err := dropTakenIn(hub, sw, func(m jetstream.Msg) { t.Errorf("the main durable worked %s after the bind", m.Subject()); _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := hub.Start(ctx, js, cred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer hub.Stop()
|
||||
heard := make(chan string, 1)
|
||||
takeIns := &oneHandler{}
|
||||
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bindTakeIns(ctx, js, cred, takeIns.handle, sw, t.Logf)
|
||||
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case s := <-heard:
|
||||
if s != "mesh.mod.mesh-delivery-indexer.event.take-in.taken" {
|
||||
t.Errorf("the takeins durable heard %s", s)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the take-in is not heard while late events wait on the main durable")
|
||||
}
|
||||
info, err := events.Consumer(ctx, cred.EventConsumerName()+"_"+takeInsSuffix)
|
||||
if err != nil {
|
||||
t.Fatalf("the takeins durable is not made: %v", err)
|
||||
}
|
||||
if f := info.CachedInfo().Config.FilterSubjects; len(f) != 1 || f[0] != pipeline.TakenInSubject {
|
||||
t.Errorf("the takeins durable filters %v, want take-in.taken alone", f)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-500 fourth addendum: a take-in.taken said before the runner's takeins durable is bound (the runner was
|
||||
// down, or the bind failed) is worked by the main durable, so it starts its pipeline; one said after the bind is worked by
|
||||
// the takeins durable alone and the main durable acknowledges it with no work. A row of
|
||||
// TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview.
|
||||
func testATakeInSaidBeforeTheBindIsStillWorked(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"}
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: cred.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const subject = "mesh.mod.mesh-delivery-indexer.event.take-in.taken"
|
||||
// Said while the runner was down: the main durable holds it, the takeins durable does not exist yet.
|
||||
if _, err := js.Publish(ctx, subject, []byte(`{"before":true}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The main durable's take-in handler is held until the takeins durable is bound, as a main durable busy with step
|
||||
// events would be: the take-in said before the bind is still worked, by its sequence, not by the moment it is heard.
|
||||
held := make(chan struct{})
|
||||
worked, heard := make(chan string, 2), make(chan string, 2)
|
||||
hub := process.NewEventHub(t.Logf)
|
||||
sw := &takeInsSwitch{}
|
||||
if err := dropTakenIn(hub, sw, func(m jetstream.Msg) { <-held; worked <- string(m.Data()); _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := hub.Start(ctx, js, cred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer hub.Stop()
|
||||
takeIns := &oneHandler{}
|
||||
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- string(m.Data()); _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bindTakeIns(ctx, js, cred, takeIns.handle, sw, t.Logf)
|
||||
if sw.from.Load() == 0 {
|
||||
t.Fatal("the takeins durable is not bound")
|
||||
}
|
||||
// Said after the bind: the takeins durable's alone.
|
||||
if _, err := js.Publish(ctx, subject, []byte(`{"after":true}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
close(held)
|
||||
select {
|
||||
case d := <-worked:
|
||||
if d != `{"before":true}` {
|
||||
t.Errorf("the main durable worked %s, want the take-in said before the bind", d)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the take-in said before the bind is worked by no durable")
|
||||
}
|
||||
select {
|
||||
case d := <-heard:
|
||||
if d != `{"after":true}` {
|
||||
t.Errorf("the takeins durable heard %s, want the take-in said after the bind", d)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the take-in said after the bind is not heard on the takeins durable")
|
||||
}
|
||||
// The main durable acknowledged the second with no work: nothing more is worked once its queue is drained.
|
||||
select {
|
||||
case d := <-worked:
|
||||
t.Errorf("the main durable worked %s after the bind", d)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
info, err := events.Consumer(ctx, cred.EventConsumerName())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ci, err := info.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ci.NumAckPending != 0 || ci.NumPending != 0 {
|
||||
t.Errorf("the main durable holds %d pending and %d unacknowledged, want none", ci.NumPending, ci.NumAckPending)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-500 fifth addendum: a takeins bind that failed at the start (here the events stream is not there yet, as
|
||||
// the runner's grant is not there before the bus composes its account) is said and bound again on the next
|
||||
// registration-changed, never on a timer. A row of TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview.
|
||||
func testATakeInsBindThatFailedIsBoundAgainOnRegistrationChanged(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"}
|
||||
sw := &takeInsSwitch{}
|
||||
heard := make(chan string, 1)
|
||||
again := bindTakeIns(ctx, js, cred, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }, sw, t.Logf)
|
||||
if sw.from.Load() != 0 {
|
||||
t.Fatal("the takeins durable is bound with no events stream")
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again() // the registration-changed heard once the bus holds the stream and the grant
|
||||
if sw.from.Load() == 0 {
|
||||
t.Fatal("the takeins durable is not bound again on registration-changed")
|
||||
}
|
||||
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-heard:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the take-in is not heard on the durable bound again")
|
||||
}
|
||||
again() // bound already: nothing is bound twice
|
||||
}
|
||||
@@ -166,6 +166,39 @@ func (t testRunsKV) Has(key string) (bool, error) {
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// askValue is an ask key's value: the key of the run the ask is of (H-A-500 addendum (1)).
|
||||
type askValue struct {
|
||||
Run string `json:"run"`
|
||||
}
|
||||
|
||||
// MarkAsk creates the ask key of ask holding run; one written already is kept.
|
||||
func (t testRunsKV) MarkAsk(ask, run string) error {
|
||||
raw, err := json.Marshal(askValue{Run: run})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := t.kv.Create(t.ctx, pipeline.AskKey(ask), raw); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunOfAsk reads the ask key of ask: the run it holds.
|
||||
func (t testRunsKV) RunOfAsk(ask string) (string, bool, error) {
|
||||
e, err := t.kv.Get(t.ctx, pipeline.AskKey(ask))
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
var v askValue
|
||||
if err := json.Unmarshal(e.Value(), &v); err != nil || v.Run == "" {
|
||||
return "", false, fmt.Errorf("the ask key of ask %s holds no run: %v", ask, err)
|
||||
}
|
||||
return v.Run, true, nil
|
||||
}
|
||||
|
||||
func (t testRunsKV) Keys() ([]string, error) {
|
||||
all, err := t.kv.Keys(t.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
@@ -416,7 +449,7 @@ func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Acco
|
||||
hearer.TestRuns, hearer.Clock, hearer.Runs, hearer.DropRun = runs, runs, settings, drop
|
||||
hearer.RunOutputs = objects
|
||||
if hearer.Ager != nil {
|
||||
hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept = runs, objects.Delete, settings.Kept
|
||||
hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept, hearer.Ager.Clock = runs, objects.Delete, settings.Kept, runs
|
||||
}
|
||||
logf("test runs: cap %d bytes, kept %s, step bound %s, untaken bound %s", settings.CapBytes, settings.Kept, settings.StepBound, settings.Untaken)
|
||||
return kv, objects, nil
|
||||
|
||||
+5
-1
@@ -75,7 +75,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}})
|
||||
// The runner's findings ride its own stream, as its manifest declares, never EVENTS: on EVENTS each one would reach its
|
||||
// main durable unhandled and write event-unhandled again, a flood a take-in on its own durable overtakes (H-A-500 (b)).
|
||||
findingsStream(t, ctx, js, module)
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -224,6 +227,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }()
|
||||
takeInsHeard(t, ctx, logs)
|
||||
return func() { stop(); <-done }
|
||||
}
|
||||
stop := start()
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -150,10 +151,17 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi
|
||||
for {
|
||||
raw, err := call("seat:"+catalog.Seat+"."+catalog.VerbRevision, in)
|
||||
if err != nil {
|
||||
// A revision the catalog answers "no such object" for is gone (design 47 H2 H-A-496): a kept mover's wait on it ends.
|
||||
if strings.Contains(err.Error(), "no such object") {
|
||||
return pipeline.Half{}, fmt.Errorf("%w: %v", pipeline.ErrRevisionGone, err)
|
||||
}
|
||||
return pipeline.Half{}, err
|
||||
}
|
||||
answer, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
if strings.Contains(said, "no such object") {
|
||||
return pipeline.Half{}, fmt.Errorf("%w: %s answered: %s", pipeline.ErrRevisionGone, catalog.VerbRevision, said)
|
||||
}
|
||||
return pipeline.Half{}, fmt.Errorf("%s answered: %s", catalog.VerbRevision, said)
|
||||
}
|
||||
var a catalog.RevisionAnswer
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -63,6 +64,19 @@ func TestCatalogRevisionIsKeptOnceRead(t *testing.T) {
|
||||
if h, err := read("main", 0); err != nil || calls != 3 || len(h.Entries) != 2 {
|
||||
t.Fatalf("the newest made calls %d, %+v, %v; want its first page only", calls, h, err)
|
||||
}
|
||||
// A revision the catalog answers "no such object" for is gone, whether refused or answered so (design 47 H2 H-A-496).
|
||||
for _, gone := range []func(string, any) (json.RawMessage, error){
|
||||
func(string, any) (json.RawMessage, error) {
|
||||
return nil, fmt.Errorf("revision main.292: no such object")
|
||||
},
|
||||
func(string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"content":[{"text":"revision main.292: no such object"}],"isError":true}`), nil
|
||||
},
|
||||
} {
|
||||
if _, err := catalogRevision(gone, newRevisionCache(1))("main", 292); !errors.Is(err, pipeline.ErrRevisionGone) {
|
||||
t.Errorf("a revision the catalog has no object of reads %v, want it gone", err)
|
||||
}
|
||||
}
|
||||
c := newRevisionCache(1)
|
||||
c.put("main", pipeline.Half{Revision: 1})
|
||||
c.put("main", pipeline.Half{Revision: 2})
|
||||
|
||||
@@ -57,7 +57,9 @@ type Ager struct {
|
||||
Runs TestRuns
|
||||
RunOutputs func(name string) error
|
||||
RunsKept time.Duration
|
||||
Now func() time.Time
|
||||
// Clock is test-runs' bound and ask keys: a run's ask keys are deleted with the run (H-A-500 addendum (1)); nil keeps them.
|
||||
Clock RunClock
|
||||
Now func() time.Time
|
||||
// Logf is the runner's journal: each delete writes a line naming the key or object and its retention.
|
||||
Logf func(string, ...any)
|
||||
}
|
||||
@@ -159,6 +161,7 @@ func (a *Ager) ageRun(r Retention, e seat.LogEntry) {
|
||||
return
|
||||
}
|
||||
a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age)
|
||||
unmarkAsks(a.Clock, run, a.logf)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,6 +96,7 @@ func (h *Hearer) putDeclined(key, old string, s seat.StepResult, a Asked, ev Ste
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("the test run %s is not written: %w", a.Run, err)
|
||||
}
|
||||
markAsk(h.Clock, a.ID, a.Run, h.logf) // H-A-500 addendum (1)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,11 +223,11 @@ func TestAReleasedVersionIsTheMoversVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H2 §785: a pipeline of two movers ends with its taken, placed and ended times and each step kind's span, the
|
||||
// Design 47 H2 §785: a pipeline of two movers ends with its sequence (H-A-504 addendum) and its taken, placed and ended times and each step kind's span, the
|
||||
// earliest start and the latest end across its movers; a kind none of whose steps started has none.
|
||||
func TestPipelineEndedCarriesEachStepKindsSpan(t *testing.T) {
|
||||
at := func(m int) *time.Time { v := time.Date(2026, 11, 22, 1, m, 0, 0, time.UTC); return &v }
|
||||
p := seat.Pipeline{Pipeline: "main.7", Started: at(0), Ended: at(30), Outcome: seat.PipelineOutcomePassed,
|
||||
p := seat.Pipeline{Pipeline: "main.7", Seq: 270, Started: at(0), Ended: at(30), Outcome: seat.PipelineOutcomePassed,
|
||||
Movers: []seat.Mover{{Module: "lib", Tier: 1}, {Module: "app", Tier: 2}},
|
||||
Steps: []seat.StepResult{
|
||||
{Module: "lib", Step: seat.StepResultStepBuild, Started: at(2), Ended: at(6)},
|
||||
@@ -236,6 +236,9 @@ func TestPipelineEndedCarriesEachStepKindsSpan(t *testing.T) {
|
||||
{Module: "app", Step: seat.StepResultStepUnit, Started: at(10)},
|
||||
{Module: "app", Step: seat.StepResultStepComposition},
|
||||
}}
|
||||
if seq := pipelineEnded(p).Seq; seq != 270 {
|
||||
t.Errorf("pipeline-ended carries seq %d, want the record's 270 (H-A-504 addendum)", seq)
|
||||
}
|
||||
ph := pipelineEnded(p).Phases
|
||||
if ph == nil || !ph.Taken.Equal(*at(0)) || !ph.Placed.Equal(*at(1)) || !ph.Ended.Equal(*at(30)) {
|
||||
t.Fatalf("phases %+v, want taken 01:00, placed 01:01, ended 01:30", ph)
|
||||
|
||||
@@ -117,7 +117,7 @@ func (q *Queue) run(key string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// next is the sequence the next waiting pipeline gets: queue.next read and written one higher by compare-and-set, from 1.
|
||||
// next is the sequence the next pipeline gets, waiting or placed: queue.next read and written one higher by compare-and-set, from 1.
|
||||
func (q *Queue) next() (int64, error) {
|
||||
for {
|
||||
raw, rev, err := q.Store.Get(seat.QueueByQueueNextKey())
|
||||
@@ -180,23 +180,26 @@ func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how
|
||||
}
|
||||
}
|
||||
|
||||
// queueStart judges a start against the bound (§748): under it, it answers true, the pipeline placed and its running key
|
||||
// written once the pipeline is (ran); over it p gets the next sequence and state waiting, with no step, and its waiting key
|
||||
// is written first, so a key whose pipeline was never written is dropped at its placing; it answers false and that key. A
|
||||
// start with no queue is placed.
|
||||
// queueStart gives p the next sequence of queue.next, whether it waits or is placed at once, so every pipeline's seq grows
|
||||
// across the pipelines of a branch and the assigner writes it as its wants' run (design 47 H2 §813 (1), H-A-504 addendum);
|
||||
// then it judges the start against the bound (§748): under it, it answers true, the pipeline placed and its running key
|
||||
// written once the pipeline is (ran); over it p gets state waiting, with no step, and its waiting key is written first, so a
|
||||
// key whose pipeline was never written is dropped at its placing; it answers false and that key. A start with no queue is
|
||||
// placed.
|
||||
func (s *Starter) queueStart(p *seat.Pipeline) (bool, string, error) {
|
||||
if s.Queue == nil {
|
||||
return true, "", nil
|
||||
}
|
||||
full, err := s.Queue.full()
|
||||
if err != nil || !full {
|
||||
return err == nil, "", err
|
||||
}
|
||||
seq, err := s.Queue.next()
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
p.Seq, p.Outcome = seq, seat.PipelineOutcomeWaiting
|
||||
p.Seq = seq
|
||||
full, err := s.Queue.full()
|
||||
if err != nil || !full {
|
||||
return err == nil, "", err
|
||||
}
|
||||
p.Outcome = seat.PipelineOutcomeWaiting
|
||||
return false, sequenceKey(seq), s.Queue.wait(seq, p.Pipeline)
|
||||
}
|
||||
|
||||
|
||||
@@ -116,8 +116,9 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) {
|
||||
take := takeIn(t, s, view)
|
||||
take(4, "app")
|
||||
first := len(asks.ids)
|
||||
if p, _, _ := mem.Get("main.4"); p.Outcome != seat.PipelineOutcomeRunning || p.Seq != 0 || first == 0 {
|
||||
t.Fatalf("the first pipeline = %s seq %d with %d asks, want running and asked", p.Outcome, p.Seq, first)
|
||||
if p, _, _ := mem.Get("main.4"); p.Outcome != seat.PipelineOutcomeRunning || p.Seq != 1 || first == 0 {
|
||||
// H-A-504 addendum: a pipeline placed at once has its sequence too
|
||||
t.Fatalf("the first pipeline = %s seq %d with %d asks, want running at 1 and asked", p.Outcome, p.Seq, first)
|
||||
}
|
||||
if _, ok := q["running.main.4"]; !ok {
|
||||
t.Fatalf("the queue = %v, want running.main.4", q)
|
||||
@@ -127,15 +128,15 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) {
|
||||
take(6, "tool")
|
||||
for i, key := range []string{"main.5", "main.6"} {
|
||||
p, _, _ := mem.Get(key)
|
||||
if p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != int64(i+1) || len(p.Steps) != 0 {
|
||||
t.Errorf("%s = %s seq %d with %d steps, want waiting at %d with none", key, p.Outcome, p.Seq, len(p.Steps), i+1)
|
||||
if p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != int64(i+2) || len(p.Steps) != 0 {
|
||||
t.Errorf("%s = %s seq %d with %d steps, want waiting at %d with none", key, p.Outcome, p.Seq, len(p.Steps), i+2)
|
||||
}
|
||||
}
|
||||
if len(asks.ids) != first {
|
||||
t.Errorf("a waiting pipeline asked %d steps", len(asks.ids)-first)
|
||||
}
|
||||
if _, ok := q["waiting.00000000000000000001"]; !ok || string(q["queue.next"].v) != "3" {
|
||||
t.Errorf("the queue = %v, want waiting.<1 in 20 digits> and queue.next 3", q)
|
||||
if _, ok := q["waiting.00000000000000000002"]; !ok || string(q["queue.next"].v) != "4" {
|
||||
t.Errorf("the queue = %v, want waiting.<2 in 20 digits> and queue.next 4", q)
|
||||
}
|
||||
if !slices.Contains(*said, "pipeline-started.main.5") || !slices.Contains(*said, "wait-held.main.5") || len(*waits) != 2 ||
|
||||
(*waits)[0].Step != seat.WaitStepPipeline || (*waits)[0].On != seat.WaitOnPipelinesAtOnce || (*waits)[0].Pipeline != "main.5" {
|
||||
|
||||
@@ -132,6 +132,11 @@ func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Move
|
||||
s.logf("%s keeps the movers revision %d of its own source %d re-pinned, %s: %d steps ended superseded, %d asked again (H-A-283, H-A-284)",
|
||||
key, half.Revision, half.Source, strings.Join(moved, ", "), len(ended), len(asks))
|
||||
s.sayRePinned(p, kept, revisions, half.Revision)
|
||||
// The re-pin says pipeline-started again with every revision its movers are kept at, so the indexer pins them (design
|
||||
// 47 H2 H-A-496 and its addendum).
|
||||
if keeps {
|
||||
s.sayStartedPipeline(p, now)
|
||||
}
|
||||
for _, st := range ended {
|
||||
if s.Say == nil {
|
||||
break
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
asks := &stepAsks{}
|
||||
var said []seat.StepEvent
|
||||
var repinned []string
|
||||
var repinned, starts []string
|
||||
var left [][]seat.StepResult
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: asks, Name: "the-runner", Now: ticking(), Revision: read,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
|
||||
@@ -77,6 +77,9 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
if ev, ok := body.(seat.RePinnedEvent); ok {
|
||||
repinned = append(repinned, fmt.Sprintf("%s %s %s %d", subject, ev.Pipeline, ev.Mover, ev.Revision))
|
||||
}
|
||||
if ev, ok := body.(seat.PipelineStarted); ok {
|
||||
starts = append(starts, fmt.Sprintf("%s %v", ev.Pipeline, ev.Kept))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Left: func(_ string, leave, drop []seat.StepResult, _ string) { left = append(left, append(leave, drop...)) }}
|
||||
@@ -133,6 +136,11 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
if kept, err := store.KeptRevisions(key); err != nil || kept["app"] != 11 {
|
||||
t.Errorf("kept-revisions = %v (%v), want app at 11", kept, err)
|
||||
}
|
||||
// design 47 H2 H-A-496 and its addendum: the start names no kept revision, the re-pin says pipeline-started again with
|
||||
// every revision its movers are kept at.
|
||||
if want := []string{key + " []", key + " [11]"}; !slices.Equal(starts, want) {
|
||||
t.Errorf("pipeline-started said %q, want %q", starts, want)
|
||||
}
|
||||
// The pin check reads the kept mover's entry at the revision it was kept at, never the pipeline's.
|
||||
if n, err := s.AskPinHeld(store, key, "app", map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26", "git.novox.be/novox/x/go": "v0.1.1"}, read); n == 0 || err != nil {
|
||||
after, _, _ := store.Get(key)
|
||||
@@ -255,6 +263,60 @@ func TestAKeptMoverWaitingOnItsRevisionSaysSo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// design 47 H2 H-A-496: a kept mover whose kept revision the catalog answers "no such object" for waits no longer: the
|
||||
// pipeline ends failed "kept revision 11 gone" at the placing, is said ended, frees its turn, and leaves its mover
|
||||
// unfinished, so it rejoins the next pipeline.
|
||||
func TestAKeptRevisionGoneEndsThePipeline(t *testing.T) {
|
||||
reg, entries, _, read := repinFixture()
|
||||
revision := func(branch string, rev int64) (Half, error) {
|
||||
if rev == 11 {
|
||||
return Half{}, fmt.Errorf("revision main.11: no such object: %w", ErrRevisionGone)
|
||||
}
|
||||
return read(branch, rev)
|
||||
}
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
var turned, ended []string
|
||||
u := memUnfinished{}
|
||||
h := &Hearer{Store: store, Name: "the-runner", Unfinished: &Unfinished{Store: u},
|
||||
Say: func(subject string, body any) error {
|
||||
if ev, ok := body.(seat.PipelineEnded); ok {
|
||||
ended = append(ended, ev.Pipeline+" "+ev.Outcome)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Turn: func(key string) { turned = append(turned, key) }}
|
||||
registered := false
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: &stepAsks{}, Name: "the-runner", Now: ticking(), Revision: revision,
|
||||
Say: func(string, any) error { return nil }, Ended: h.EndedAtPlacing,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
|
||||
key := Key("main", 10)
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: 41, Revision: 10, Entries: entries,
|
||||
Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
if n, err := s.Revised(revisionEvent(11), read); n != 0 || err != nil {
|
||||
t.Fatalf("the re-pin started %d (%v)", n, err)
|
||||
}
|
||||
registered = true
|
||||
lagging := func(string) (Half, bool) { return Half{Revision: 10, Source: 41, Entries: entries}, true }
|
||||
if _, err := s.Registered(store, reg, lagging); err != nil {
|
||||
t.Fatalf("the placing: %v", err)
|
||||
}
|
||||
p, _, err := store.Get(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Outcome != seat.PipelineOutcomeFailed || p.Ended == nil || !strings.HasPrefix(p.Why, "kept revision 11 gone") {
|
||||
t.Fatalf("the pipeline = %s, %q, ended %v: want failed with kept revision 11 gone", p.Outcome, p.Why, p.Ended)
|
||||
}
|
||||
if !slices.Equal(ended, []string{key + " failed"}) || !slices.Equal(turned, []string{key}) {
|
||||
t.Fatalf("said ended %v, turns freed %v: want %s's end said and its turn freed", ended, turned, key)
|
||||
}
|
||||
if _, held := u[UnfinishedKey("main", "app")]; !held {
|
||||
t.Fatalf("app is not left unfinished, so it rejoins no pipeline: %v", u)
|
||||
}
|
||||
}
|
||||
|
||||
// rawKept reads a memRaw as the reads do.
|
||||
type rawKept struct{ r *memRaw }
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
@@ -54,6 +55,81 @@ type RunClock interface {
|
||||
Has(key string) (bool, error)
|
||||
// Keys are test-runs' keys.
|
||||
Keys() ([]string, error)
|
||||
// MarkAsk writes the ask key of ask (AskKey) holding run, create-only: one written already is kept (H-A-500 addendum (1)).
|
||||
MarkAsk(ask, run string) error
|
||||
// RunOfAsk reads the ask key of ask: the key of the run it holds; ok false when it is not written.
|
||||
RunOfAsk(ask string) (run string, ok bool, err error)
|
||||
}
|
||||
|
||||
// AskPrefix begins the ask key of a test run's ask in test-runs, `ask.<ask>`, holding the run's key: written with each ask
|
||||
// a run is given and deleted with the run, so an ask-taken or step-progress finds its run in one read and never lists the
|
||||
// pipelines (design 47 H2 H-A-500 addendum (1)).
|
||||
const AskPrefix = "ask."
|
||||
|
||||
// AskKey is the ask key of ask.
|
||||
func AskKey(ask string) string { return AskPrefix + ask }
|
||||
|
||||
// IsRunAsk says whether a key of test-runs is an ask key rather than a run.
|
||||
func IsRunAsk(key string) bool { return strings.HasPrefix(key, AskPrefix) }
|
||||
|
||||
// markAsk writes the ask key of a run's ask; one not written is said, and the ask's events are then found by the pipelines.
|
||||
func markAsk(clock RunClock, ask, run string, logf func(string, ...any)) {
|
||||
if clock == nil || ask == "" || run == "" {
|
||||
return
|
||||
}
|
||||
if err := clock.MarkAsk(ask, run); err != nil && logf != nil {
|
||||
logf("the ask key of ask %s of the test run %s is not written, so its events are found by the pipelines: %v", ask, run, err)
|
||||
}
|
||||
}
|
||||
|
||||
// unmarkAsks deletes, with the run under run, every ask key holding it (H-A-500 addendum (1)): the ask it names now and each
|
||||
// it was given before (a rerun, a declined step asked again). A key not read or not deleted is said. The Hearer calls it
|
||||
// when a run's last reader leaves, the Ager when a run is aged, so no ask key outlives its run.
|
||||
func unmarkAsks(clock RunClock, run string, logf func(string, ...any)) {
|
||||
if clock == nil {
|
||||
return
|
||||
}
|
||||
all, err := clock.Keys()
|
||||
if err != nil {
|
||||
logf("the ask keys of the deleted test run %s are not listed, so they are kept: %v", run, err)
|
||||
return
|
||||
}
|
||||
for _, k := range all {
|
||||
if !IsRunAsk(k) {
|
||||
continue
|
||||
}
|
||||
if held, ok, err := clock.RunOfAsk(strings.TrimPrefix(k, AskPrefix)); err != nil || !ok || held != run {
|
||||
continue
|
||||
}
|
||||
if err := clock.Unbound(k); err != nil {
|
||||
logf("the ask key %s of the deleted test run %s is not deleted: %v", k, run, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// LateEveryLine is how many late events of ended runs' asks one journal line says (H-A-500 (a)).
|
||||
const LateEveryLine = 100
|
||||
|
||||
// LateRunAsk says whether ask is the ask of a test run that has ended, or whose run was asked again under another ask
|
||||
// since, read from its ask key and its run, never from the pipelines (design 47 H2 H-A-500 (a) and its addenda): such an
|
||||
// event is acknowledged at once and moves nothing, which replaces H-A-311 (2)'s placing and H-A-238 (2)'s clearing for
|
||||
// it. One journal line says every LateEveryLine of them. An ask with no ask key, or whose run is gone, is not judged here.
|
||||
func (h *Hearer) LateRunAsk(event, ask string) bool {
|
||||
if h.TestRuns == nil || h.Clock == nil || ask == "" {
|
||||
return false
|
||||
}
|
||||
run, ok, err := h.Clock.RunOfAsk(ask)
|
||||
if err != nil || !ok {
|
||||
return false
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
if err != nil || (r.Ended == nil && r.Ask == ask) {
|
||||
return false
|
||||
}
|
||||
if n := atomic.AddInt64(&h.lateRuns, 1); n%LateEveryLine == 0 {
|
||||
h.logf("%d late events of ended test runs' asks acknowledged at once, moving nothing; the last the %s of ask %s of the test run %s (H-A-500)", n, event, ask, run)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ParseRunBound reads a key of test-runs as a run's bound: the run's key and whether it is the untaken bound.
|
||||
@@ -84,6 +160,13 @@ func (h *Hearer) runAndPipelineOfAsk(ask string) (string, string, bool) {
|
||||
if h.TestRuns == nil || ask == "" {
|
||||
return "", "", false
|
||||
}
|
||||
// The ask key names the run in one read (H-A-500 addendum (1)); the pipeline is then left unnamed, the run's readers
|
||||
// naming every pipeline that reads it, the asking one among them.
|
||||
if h.Clock != nil {
|
||||
if run, ok, err := h.Clock.RunOfAsk(ask); err == nil && ok {
|
||||
return run, "", true
|
||||
}
|
||||
}
|
||||
pkeys, err := allKeys(h.Store)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
@@ -254,7 +337,7 @@ func (h *Hearer) TakeUp() (int, error) {
|
||||
ended := 0
|
||||
var first error
|
||||
for _, run := range all {
|
||||
if IsRunBound(run) {
|
||||
if IsRunBound(run) || IsRunAsk(run) {
|
||||
continue
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
|
||||
@@ -347,6 +347,27 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A
|
||||
return nil, ended
|
||||
}
|
||||
|
||||
// keptGone ends p failed at its placing, "kept revision <r> gone", when the catalog no longer holds revision r a mover of
|
||||
// it is kept at (design 47 H2 H-A-496): each step not ended ends not asked, so its movers, whose steps never ran, are left
|
||||
// unfinished and rejoin the next pipeline (§786), and its end frees its turn under pipelines-at-once. It answers the steps
|
||||
// it ended, none when it had none; the pipeline's Ended says it ended.
|
||||
func keptGone(p *seat.Pipeline, r int64, err error, now time.Time) []int {
|
||||
why := fmt.Sprintf("kept revision %d gone", r)
|
||||
ended := []int{}
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
st.Outcome, st.Why, st.Ended, st.Ask, st.Shared = seat.StepResultOutcomeNotAsked, "the pipeline failed: "+why, &now, "", nil
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
}
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, fmt.Sprintf("%s (%v)", why, err), &now
|
||||
p.EndCount++
|
||||
return ended
|
||||
}
|
||||
|
||||
// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the
|
||||
// pipeline's end, through Ended; with none given it is said in the journal alone.
|
||||
func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error {
|
||||
@@ -416,18 +437,36 @@ func (s *Starter) create(key string, p seat.Pipeline) error {
|
||||
return s.Pipelines.Create(key, raw)
|
||||
}
|
||||
|
||||
// started is a pipeline's pipeline-started (§746): its key, branch, source, call, revision and movers.
|
||||
func started(p seat.Pipeline, at time.Time) seat.PipelineStarted {
|
||||
// started is a pipeline's pipeline-started (§746): its key, branch, source, call, revision and movers, and kept, every
|
||||
// catalog revision its movers are kept at (design 47 H2 H-A-496), so the indexer pins each while the pipeline runs.
|
||||
func started(p seat.Pipeline, at time.Time, kept []int64) seat.PipelineStarted {
|
||||
return seat.PipelineStarted{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call,
|
||||
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run}
|
||||
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run, Kept: kept}
|
||||
}
|
||||
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<revision> (§746).
|
||||
// keptList is every distinct revision of kept-revisions, in order (H-A-496).
|
||||
func keptList(kept map[string]int64) []int64 {
|
||||
var out []int64
|
||||
for _, r := range kept {
|
||||
if r > 0 && !slices.Contains(out, r) {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<revision> (§746), with the revisions its movers are kept at as
|
||||
// its record's kept-revisions holds them (H-A-496): a re-pin says it again with the full list (H-A-496 addendum).
|
||||
func (s *Starter) sayStartedPipeline(p seat.Pipeline, at time.Time) {
|
||||
if s.Say == nil {
|
||||
return
|
||||
}
|
||||
if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at)); err != nil {
|
||||
kept, err := keptOfStore(s.Store, p.Pipeline)
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its pipeline-started names no kept revision: %v", p.Pipeline, err)
|
||||
}
|
||||
if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at, keptList(kept))); err != nil {
|
||||
s.logf("pipeline-started of %s is not said: %v", p.Pipeline, err)
|
||||
}
|
||||
}
|
||||
@@ -445,6 +484,8 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its movers are placed as movers not kept: %v", p.Pipeline, err)
|
||||
}
|
||||
var gone int64
|
||||
var goneWhy error
|
||||
all := entries
|
||||
for i, m := range p.Movers {
|
||||
// A mover that left for a newer pipeline is placed there, never here (H-A-223, H-A-361).
|
||||
@@ -460,7 +501,13 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
entries := all
|
||||
if rev := kept[m.Module]; rev > 0 {
|
||||
var err error
|
||||
if entries, err = s.keptEntries(p.Branch, rev); err != nil {
|
||||
if entries, err = s.keptEntries(p.Branch, rev); errors.Is(err, ErrRevisionGone) {
|
||||
// A kept revision the catalog no longer holds is never read: the pipeline ends failed (H-A-496).
|
||||
if gone == 0 {
|
||||
gone, goneWhy = rev, err
|
||||
}
|
||||
continue
|
||||
} else if err != nil {
|
||||
// It waits for its kept revision to be read, said once and carried on the mover (design 47 H2 §787).
|
||||
if m.Unplaced != seat.MoverUnplacedKeptRevisionUnread || m.Revision != rev {
|
||||
s.logf("%s: %s waits for its kept revision %d to be read (%v)", p.Pipeline, m.Module, rev, err)
|
||||
@@ -520,6 +567,9 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
}
|
||||
}
|
||||
}
|
||||
if gone > 0 {
|
||||
return nil, unregistered, keptGone(p, gone, goneWhy, now), notAsked
|
||||
}
|
||||
if steps, ended := cycled(p, waits, unregistered, reg, all, now, s.Name); ended {
|
||||
return nil, unregistered, steps, notAsked
|
||||
} else {
|
||||
@@ -702,7 +752,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
}
|
||||
s.ran(key)
|
||||
s.sayNotAsked(p, notAsked)
|
||||
if len(ended) > 0 {
|
||||
if len(ended) > 0 || p.Ended != nil {
|
||||
return true, s.endedAtPlacing(p, ended)
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
@@ -878,16 +928,19 @@ func (s *Starter) placeWaiting(store Updater, due func(seat.Pipeline) bool, reg
|
||||
if err != nil {
|
||||
note(err)
|
||||
}
|
||||
if n == 0 {
|
||||
// A pipeline its placing ended (a kept revision gone, H-A-496) is said ended though it placed no step.
|
||||
if n == 0 && ended == nil {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
s.logf("%s: %d steps placed from %s (A-327)", key, n, from)
|
||||
// A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start,
|
||||
// said already, is said again harmlessly.
|
||||
s.sayStartedPipeline(p, now)
|
||||
if n > 0 {
|
||||
s.logf("%s: %d steps placed from %s (A-327)", key, n, from)
|
||||
// A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start,
|
||||
// said already, is said again harmlessly.
|
||||
s.sayStartedPipeline(p, now)
|
||||
}
|
||||
s.sayNotAsked(p, notAsked)
|
||||
if len(ended) > 0 {
|
||||
if ended != nil {
|
||||
if err := s.endedAtPlacing(p, ended); err != nil {
|
||||
note(err)
|
||||
}
|
||||
@@ -933,7 +986,7 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
|
||||
s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m)
|
||||
s.registrationMissing(m, key)
|
||||
}
|
||||
if len(p.Steps) == before {
|
||||
if len(p.Steps) == before && p.Ended == nil {
|
||||
// A mover that now waits for its kept revision, or no longer does, is written so (design 47 H2 §787).
|
||||
if unplacedMarks(p) == marks {
|
||||
return p, nil, nil, nil, 0, nil
|
||||
|
||||
@@ -194,6 +194,9 @@ type Hearer struct {
|
||||
// queue (Starter.PutAgain); with either nil a declined step ends could not run.
|
||||
AskDeclined func(p seat.Pipeline, s seat.StepResult, now time.Time) (*Asked, error)
|
||||
PutAgain func(key string, a Asked) error
|
||||
|
||||
// lateRuns counts the late events of ended runs' asks acknowledged at once (LateRunAsk, H-A-500 (a)).
|
||||
lateRuns int64
|
||||
}
|
||||
|
||||
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
|
||||
@@ -690,7 +693,8 @@ func stepsOf(p seat.Pipeline, module string) []seat.StepResult {
|
||||
return out
|
||||
}
|
||||
|
||||
// pipelineEnded is p's pipeline-ended (design 47 H §746): its key, branch, source, call, revision and outcome, and per
|
||||
// pipelineEnded is p's pipeline-ended (design 47 H §746): its key, branch, source, call, revision, sequence (H2 §813 (1),
|
||||
// H-A-504 addendum) and outcome, and per
|
||||
// mover, in the delivery order of its tiers, its module, fingerprint, commit, widens (H-A-119), released version, step results
|
||||
// and tier; and the movers its end left unfinished (design 47 H2 §786).
|
||||
func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
|
||||
@@ -701,7 +705,7 @@ func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
|
||||
}
|
||||
sort.SliceStable(movers, func(i, j int) bool { return movers[i].Tier < movers[j].Tier })
|
||||
return seat.PipelineEnded{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call, Revision: p.Revision,
|
||||
Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started, Ended: p.Ended, EndCount: p.EndCount, Movers: movers,
|
||||
Seq: p.Seq, Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started, Ended: p.Ended, EndCount: p.EndCount, Movers: movers,
|
||||
Phases: phasesOf(p), Unfinished: unfinishedOf(p)}
|
||||
}
|
||||
|
||||
|
||||
@@ -250,6 +250,8 @@ func (s *Starter) started(a Asked, write func() error) error {
|
||||
if err := write(); err != nil {
|
||||
return err
|
||||
}
|
||||
// The run's ask gets its ask key, so its events find the run in one read (H-A-500 addendum (1)).
|
||||
markAsk(s.Clock, a.ID, a.Run, s.logf)
|
||||
if s.Clock != nil {
|
||||
untaken := s.Runs.Untaken
|
||||
if untaken <= 0 {
|
||||
|
||||
@@ -38,7 +38,7 @@ func (r RunsRead) TestRuns(in seat.TestRunsInput) (seat.TestRunsAnswer, error) {
|
||||
sort.Strings(all)
|
||||
a := seat.TestRunsAnswer{Runs: []seat.TestRun{}}
|
||||
for _, key := range all {
|
||||
if IsRunBound(key) || (in.Candidate != "" && !strings.HasPrefix(key, in.Candidate+".")) {
|
||||
if IsRunBound(key) || IsRunAsk(key) || (in.Candidate != "" && !strings.HasPrefix(key, in.Candidate+".")) {
|
||||
continue
|
||||
}
|
||||
raw, err := r.Runs.Value(key)
|
||||
@@ -165,6 +165,7 @@ func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.Test
|
||||
for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} {
|
||||
_ = h.Clock.Unbound(k)
|
||||
}
|
||||
unmarkAsks(h.Clock, in.Key, h.logf)
|
||||
}
|
||||
gone := r.Answer()
|
||||
gone.Key = in.Key
|
||||
|
||||
@@ -20,6 +20,8 @@ type clockMem struct {
|
||||
asks map[string]string
|
||||
revs map[string]uint64
|
||||
rev uint64
|
||||
// askRuns are the ask keys, each holding its run (H-A-500 addendum (1)).
|
||||
askRuns map[string]string
|
||||
// before is called once at the next Bound, a write that comes between its caller's read and its write.
|
||||
before func()
|
||||
}
|
||||
@@ -40,7 +42,11 @@ func (c *clockMem) Bounded(key string) (string, uint64, bool, error) {
|
||||
_, ok := c.bound[key]
|
||||
return c.asks[key], c.revs[key], ok, nil
|
||||
}
|
||||
func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil }
|
||||
func (c *clockMem) Unbound(key string) error {
|
||||
delete(c.bound, key)
|
||||
delete(c.askRuns, key)
|
||||
return nil
|
||||
}
|
||||
func (c *clockMem) UnboundAt(key string, rev uint64) error {
|
||||
if c.revs[key] == rev {
|
||||
delete(c.bound, key)
|
||||
@@ -48,6 +54,20 @@ func (c *clockMem) UnboundAt(key string, rev uint64) error {
|
||||
return nil
|
||||
}
|
||||
func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil }
|
||||
func (c *clockMem) MarkAsk(ask, run string) error {
|
||||
if _, ok := c.askRuns[AskKey(ask)]; ok {
|
||||
return nil
|
||||
}
|
||||
if c.askRuns == nil {
|
||||
c.askRuns = map[string]string{}
|
||||
}
|
||||
c.askRuns[AskKey(ask)] = run
|
||||
return nil
|
||||
}
|
||||
func (c *clockMem) RunOfAsk(ask string) (string, bool, error) {
|
||||
run, ok := c.askRuns[AskKey(ask)]
|
||||
return run, ok, nil
|
||||
}
|
||||
func (c *clockMem) Keys() ([]string, error) {
|
||||
var out []string
|
||||
for k := range c.runs {
|
||||
@@ -56,6 +76,9 @@ func (c *clockMem) Keys() ([]string, error) {
|
||||
for k := range c.bound {
|
||||
out = append(out, k)
|
||||
}
|
||||
for k := range c.askRuns {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -245,7 +268,7 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
|
||||
f.objects["out-1"] = []byte("ok")
|
||||
now := agedNow
|
||||
a := &Ager{Runs: f.runs, RunOutputs: f.objects.Delete, RunsKept: 7 * 24 * time.Hour, Pipelines: agedDeleter{f.store},
|
||||
Aged: agedState{}, Now: func() time.Time { return now },
|
||||
Clock: f.clock, Aged: agedState{}, Now: func() time.Time { return now },
|
||||
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
|
||||
if from > 1 {
|
||||
return nil, nil
|
||||
@@ -257,11 +280,18 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
|
||||
if _, _, err := f.runs.Get(run); err != nil {
|
||||
t.Fatal("a run is aged before test-runs-kept passed")
|
||||
}
|
||||
if held, ok, _ := f.clock.RunOfAsk(unit.Ask); !ok || held != run {
|
||||
t.Fatalf("the ask key of %s holds %q before the run is aged, want %s", unit.Ask, held, run)
|
||||
}
|
||||
now = now.Add(2 * 24 * time.Hour)
|
||||
a.Age()
|
||||
if _, _, err := f.runs.Get(run); err != ErrNoRun || len(f.objects) != 0 {
|
||||
t.Errorf("the run past test-runs-kept is %v, objects %v; want both deleted", err, f.objects)
|
||||
}
|
||||
// The ask key is deleted with the aged run (H-A-500 addendum (1)), as it is when the run's last reader leaves.
|
||||
if _, ok, _ := f.clock.RunOfAsk(unit.Ask); ok {
|
||||
t.Errorf("the ask key of %s outlives its aged run", unit.Ask)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -631,6 +661,44 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
|
||||
t.Errorf("the late step-progress placed in %v, want %s's waiting movers placed", released, p.Pipeline)
|
||||
}
|
||||
})
|
||||
// H-A-500 (a) and its addenda: a late ask-taken or step-progress of an ended run's ask is judged from its ask key and
|
||||
// its run alone, never the pipelines, one journal line per hundred; the ask key is deleted with the run.
|
||||
t.Run("a late event of an ended run's ask reads no pipeline", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
if got, ok, _ := f.clock.RunOfAsk(unit.Ask); !ok || got != run {
|
||||
t.Fatalf("the ask key of %s holds %q, %v; want the run %s", unit.Ask, got, ok, run)
|
||||
}
|
||||
if f.h.LateRunAsk("step-progress", unit.Ask) {
|
||||
t.Fatal("the ask of a running run is judged late")
|
||||
}
|
||||
if err := f.h.EndRun(run, seat.TestRun{Ask: unit.Ask, Outcome: seat.StepResultOutcomePassed}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := f.h.Store
|
||||
f.h.Store, f.logged = nil, nil // a read of the pipelines would panic
|
||||
for i := 0; i < 2*LateEveryLine; i++ {
|
||||
if !f.h.LateRunAsk("ask-taken", unit.Ask) {
|
||||
t.Fatalf("event %d of the ended run's ask is not judged late", i)
|
||||
}
|
||||
}
|
||||
if f.h.LateRunAsk("ask-taken", "anaskofnorun") {
|
||||
t.Error("an ask with no ask key is judged late")
|
||||
}
|
||||
if len(f.logged) != 2 || !strings.Contains(f.logged[1], "200 late events") {
|
||||
t.Errorf("journal %q; want one line per hundred late events", f.logged)
|
||||
}
|
||||
f.h.Store = store
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, Acts: &actLog{}}
|
||||
if _, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: run, Why: "gone"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok, _ := f.clock.RunOfAsk(unit.Ask); ok {
|
||||
t.Error("the ask key is kept after its run is deleted")
|
||||
}
|
||||
})
|
||||
// A run's end is written as its step-ended is heard, before the finding the step says (A-470o).
|
||||
t.Run("the run ends before the step's finding is written", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
|
||||
@@ -27,6 +27,10 @@ const (
|
||||
IdleAfter = 24 * time.Hour
|
||||
// FullPercent is the fill of the objects past which the receiver says they fill.
|
||||
FullPercent = 80
|
||||
// TakeOverAfter is how long a transfer's last part must lie behind it before another caller opening it with the
|
||||
// same digest takes it over (design 47 H2 H-A-481): a sender restarted mid-transfer goes on, a live one is not
|
||||
// pushed aside.
|
||||
TakeOverAfter = 2 * time.Minute
|
||||
)
|
||||
|
||||
// Want is one part an opening names: its digest (sha256:<hex>) and its size.
|
||||
@@ -117,12 +121,17 @@ type Receiver[M any] struct {
|
||||
// Bound reads the cap the objects' binding carries (BusCap): the bus's own bound, which then counts over Cap.
|
||||
// Optional; an error or a zero falls back to Cap.
|
||||
Bound func() (uint64, error)
|
||||
// MaxChunk, Idle and Full override MaxReceived, IdleAfter and FullPercent when set.
|
||||
// MaxChunk, Idle, TakeOver and Full override MaxReceived, IdleAfter, TakeOverAfter and FullPercent when set.
|
||||
MaxChunk int
|
||||
Idle time.Duration
|
||||
TakeOver time.Duration
|
||||
Full uint64
|
||||
// Holds says the holder already holds a part, which then counts as held at the open and is not sent. Optional.
|
||||
Holds func(t Transfer[M], digest string) bool
|
||||
// HoldsWhole says the holder holds the file itself the transfer brings (its manifest or reference at the name and
|
||||
// version its shape names), never from its parts alone (design 47 H2 H-A-481 second addendum): a reopen that lacks
|
||||
// nothing is then dropped without a second store. Optional: nil never drops, and the reopen finishes as usual.
|
||||
HoldsWhole func(t Transfer[M]) bool
|
||||
// Check is the caller's check of a whole transfer before anything is stored; its error is the refusal's reason.
|
||||
// Optional.
|
||||
Check func(t Transfer[M]) error
|
||||
@@ -172,6 +181,13 @@ func (r *Receiver[M]) CapNow() uint64 {
|
||||
return r.Cap
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) takeOver() time.Duration {
|
||||
if r.TakeOver > 0 {
|
||||
return r.TakeOver
|
||||
}
|
||||
return TakeOverAfter
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) fullPct() uint64 {
|
||||
if r.Full > 0 {
|
||||
return r.Full
|
||||
@@ -180,8 +196,12 @@ func (r *Receiver[M]) fullPct() uint64 {
|
||||
}
|
||||
|
||||
// Open opens a transfer for opener, or opens it again: a transfer already open answers what it lacks, so a cut transfer
|
||||
// goes on from there, and only its opener may open it again. A new transfer that would pass the cap is refused; parts
|
||||
// the holder holds count as held; when nothing is lacked the transfer ends at once.
|
||||
// goes on from there. A transfer is finished or taken over, never left open (design 47 H2 H-A-481 and its addendum):
|
||||
// one opened again with nothing lacked (its finish was cut) is finished, by its opener or by another caller with the
|
||||
// same digest, once each part taken in it is read back and hashes to its digest (a part that does not is lacked again),
|
||||
// and dropped without a second store when the holder already holds the file itself (HoldsWhole); one that lacks a part is another
|
||||
// caller's only once its last part is older than TakeOver, and is refused to it before. A new transfer that would pass the cap is refused; parts the holder holds count as held; when nothing is
|
||||
// lacked the transfer ends at once, its idle bound armed before the finish, so a cut finish never leaves it open.
|
||||
func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) {
|
||||
switch {
|
||||
case strings.TrimSpace(id) == "" || strings.ContainsAny(id, "/. *>"):
|
||||
@@ -197,8 +217,33 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
if t, found, err := r.Transfers.Get(id); err != nil {
|
||||
return Received{}, err
|
||||
} else if found {
|
||||
if t.Opener != opener {
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id)
|
||||
if len(t.Lacks()) == 0 {
|
||||
if r.heldWhole(t) {
|
||||
// The cut came after the store: the holder holds the file, so it is not stored a second time.
|
||||
if _, err := r.Drop(id); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s is stored already and cannot be dropped: %w", id, err)
|
||||
}
|
||||
return Received{ID: id, Ended: true}, nil
|
||||
}
|
||||
if !r.verify(&t) {
|
||||
return r.finishArmed(t)
|
||||
}
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err)
|
||||
}
|
||||
r.tell(t.FillAtOpen)
|
||||
}
|
||||
switch {
|
||||
case len(t.Lacks()) == 0:
|
||||
return r.finishArmed(t)
|
||||
case t.Opener != opener && r.now().Sub(t.Last) < r.takeOver():
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller, whose last part came %s ago (taken over after %s)",
|
||||
opener, id, r.now().Sub(t.Last).Round(time.Second), r.takeOver())
|
||||
case t.Opener != opener:
|
||||
t.Opener = opener
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be taken over by %s: %w", id, opener, err)
|
||||
}
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
@@ -227,11 +272,53 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
}
|
||||
r.tell(used)
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
return r.finishArmed(t)
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
|
||||
// heldWhole says the holder already holds the file t brings (HoldsWhole), so a finish cut after its store stores nothing
|
||||
// again; parts held alone never say it (H-A-481 second addendum), since a publish cut before its store must still be
|
||||
// said as published or refused.
|
||||
func (r *Receiver[M]) heldWhole(t Transfer[M]) bool {
|
||||
return r.HoldsWhole != nil && r.HoldsWhole(t)
|
||||
}
|
||||
|
||||
// verify reads each part taken in t back from the objects and hashes it against its digest, as Take does at a part's
|
||||
// last chunk (H-A-481 addendum): a finish cut after its chunks were dropped reads them absent, which the runtime's
|
||||
// object read answers as empty. A part that fails has its chunks dropped and is lacked again. It answers whether any
|
||||
// part failed, t changed with it.
|
||||
func (r *Receiver[M]) verify(t *Transfer[M]) bool {
|
||||
failed := false
|
||||
for _, p := range t.Parts {
|
||||
if !t.Taken(p.Digest) || !slices.Contains(t.Held, p.Digest) {
|
||||
continue
|
||||
}
|
||||
whole, err := r.part(*t, p.Digest)
|
||||
if err == nil && "sha256:"+sum(whole) == p.Digest {
|
||||
continue
|
||||
}
|
||||
r.dropChunks(*t, p.Digest)
|
||||
delete(t.Chunks, p.Digest)
|
||||
t.Held = slices.DeleteFunc(t.Held, func(d string) bool { return d == p.Digest })
|
||||
failed = true
|
||||
}
|
||||
if failed {
|
||||
r.reanchor(t)
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
// finishArmed finishes a transfer that lacks nothing at an open, its idle bound armed first (H-A-481): a finish cut
|
||||
// by a restart leaves a kept transfer its key still drops, and the next open finishes it. A key not written fails the
|
||||
// open loudly before the finish, the transfer kept for the next open.
|
||||
func (r *Receiver[M]) finishArmed(t Transfer[M]) (Received, error) {
|
||||
if err := r.arm(t.ID); err != nil {
|
||||
return Received{}, err
|
||||
}
|
||||
return Received{ID: t.ID, Ended: true}, r.finish(t)
|
||||
}
|
||||
|
||||
// arm bounds an open transfer by its idle key. A key that cannot be written fails the open loudly (the transfer is
|
||||
// kept, so the opener's next open of it answers what is held and arms it again).
|
||||
func (r *Receiver[M]) arm(id string) error {
|
||||
|
||||
@@ -289,6 +289,106 @@ func aHolderTakesAFileInParts(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// Design 47 H2 H-A-481: a transfer is finished or taken over, never left open. A finish cut by a restart leaves the
|
||||
// transfer kept with every part held and its chunks in the objects (cut): the opener's next open, or another caller's
|
||||
// with the same digest, finishes it; a transfer that lacks a part is another caller's once its last part is older
|
||||
// than TakeOverAfter, refused to it before; and a transfer that lacks nothing at its open is armed before its finish.
|
||||
t.Run("a transfer is finished or taken over, never left open", func(t *testing.T) {
|
||||
var order []string
|
||||
pkg.Arm = func(id string, _ time.Duration) error { order = append(order, "arm "+id); return nil }
|
||||
store := pkg.Store
|
||||
defer func() { pkg.Arm, pkg.Holds, pkg.HoldsWhole, pkg.Store = nil, nil, nil, store }()
|
||||
// cut leaves c1 as a cut finish does: "whole" all chunks there, "lacked" a part short, "gone" its chunks dropped
|
||||
// before the cut, "wrong" a chunk that does not hash.
|
||||
cut := func(id, how string) {
|
||||
tr := Transfer[packageShape]{ID: id, Opener: "builder", Parts: []Want{part}, Chunks: map[string]int{part.Digest: 1},
|
||||
Held: []string{part.Digest}, Opened: now, Last: now, Shape: packageShape{"m", "1"}}
|
||||
switch how {
|
||||
case "whole":
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = blob
|
||||
case "lacked":
|
||||
tr.Held = nil
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = blob[:1000]
|
||||
case "wrong":
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = []byte("other")
|
||||
}
|
||||
if err := pkg.Transfers.Put(tr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name, opener, how string
|
||||
after, takeOver time.Duration
|
||||
storeHolds string // "parts": Holds answers every part; "file": HoldsWhole answers the file
|
||||
want string // "stored", "dropped", "lacked again", "lacks" (taken over) or the refusal's words
|
||||
}{
|
||||
{name: "the opener opens a cut finish again", opener: "builder", how: "whole", want: "stored"},
|
||||
{name: "another caller opens a cut finish", opener: "other", how: "whole", want: "stored"},
|
||||
{name: "a cut after the chunks were dropped", opener: "builder", how: "gone", want: "lacked again"},
|
||||
{name: "a cut finish whose chunk does not hash", opener: "builder", how: "wrong", want: "lacked again"},
|
||||
{name: "a cut after the store: the holder holds the file", opener: "builder", how: "whole", storeHolds: "file", want: "dropped"},
|
||||
{name: "a cut before the store: the holder holds the parts, not the file", opener: "builder", how: "whole", storeHolds: "parts", want: "stored"},
|
||||
{name: "another caller while the opener sends", opener: "other", how: "lacked", after: time.Minute, want: "another caller, whose last part came 1m0s ago"},
|
||||
{name: "another caller once the last part is two minutes old", opener: "other", how: "lacked", after: TakeOverAfter, want: "lacks"},
|
||||
{name: "the receiver's own takeover bound", opener: "other", how: "lacked", after: time.Minute, takeOver: 30 * time.Second, want: "lacks"},
|
||||
} {
|
||||
pkgLog = nil
|
||||
pkg.TakeOver = c.takeOver
|
||||
pkg.Holds, pkg.HoldsWhole = nil, nil
|
||||
switch c.storeHolds {
|
||||
case "parts":
|
||||
pkg.Holds = func(Transfer[packageShape], string) bool { return true }
|
||||
case "file":
|
||||
pkg.HoldsWhole = func(tr Transfer[packageShape]) bool { return tr.Shape == packageShape{"m", "1"} }
|
||||
}
|
||||
cut("c1", c.how)
|
||||
now = now.Add(c.after)
|
||||
got, err := pkg.Open("c1", c.opener, []Want{part}, packageShape{"m", "1"})
|
||||
kept, open, _ := pkg.Transfers.Get("c1")
|
||||
switch c.want {
|
||||
case "stored":
|
||||
if err != nil || !got.Ended || open || len(pkgLog) != 1 || pkgLog[0].outcome != "stored" || string(pkgLog[0].parts[part.Digest]) != string(blob) {
|
||||
t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it finished and stored", c.name, got, err, open, pkgLog)
|
||||
}
|
||||
case "dropped":
|
||||
if err != nil || !got.Ended || open || len(pkgLog) != 0 {
|
||||
t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it dropped, not stored again", c.name, got, err, open, pkgLog)
|
||||
}
|
||||
case "lacked again":
|
||||
if err != nil || got.Ended || !open || len(pkgLog) != 0 || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) || kept.Taken(part.Digest) {
|
||||
t.Errorf("%s: open = %+v, %v, log %+v, kept %+v; want the part lacked again, nothing stored", c.name, got, err, pkgLog, kept)
|
||||
}
|
||||
_, _ = pkg.Drop("c1")
|
||||
case "lacks":
|
||||
if err != nil || got.Ended || !open || kept.Opener != c.opener || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) {
|
||||
t.Errorf("%s: open = %+v, %v, opener %q; want it taken over by %s, lacking the part", c.name, got, err, kept.Opener, c.opener)
|
||||
}
|
||||
if r, err := pkg.Take("c1", c.opener, part.Digest, 1, true, blob[1000:], nil); err != nil || !r.Ended {
|
||||
t.Errorf("%s: the new opener's last chunk = %+v, %v", c.name, r, err)
|
||||
}
|
||||
default:
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) || !open || kept.Opener != "builder" {
|
||||
t.Errorf("%s: open = %v, opener %q; want refused with %q, the transfer kept", c.name, err, kept.Opener, c.want)
|
||||
}
|
||||
_, _ = pkg.Drop("c1")
|
||||
}
|
||||
if len(objects) != 0 {
|
||||
t.Errorf("%s: %d objects left", c.name, len(objects))
|
||||
}
|
||||
}
|
||||
pkg.TakeOver = 0
|
||||
order = nil
|
||||
pkg.Holds = func(Transfer[packageShape], string) bool { return true }
|
||||
pkg.Store = func(tr Transfer[packageShape], _ map[string][]byte) error {
|
||||
order = append(order, "store "+tr.ID)
|
||||
return nil
|
||||
}
|
||||
if got, err := pkg.Open("c2", "builder", []Want{part}, packageShape{"m", "1"}); err != nil || !got.Ended ||
|
||||
!reflect.DeepEqual(order, []string{"arm c2", "store c2"}) {
|
||||
t.Errorf("a transfer lacking nothing at its open = %+v, %v, %v; want its idle bound armed before its finish", got, err, order)
|
||||
}
|
||||
})
|
||||
|
||||
// An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the
|
||||
// rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the
|
||||
// binding carries (Bound) counts over Cap.
|
||||
|
||||
@@ -84,7 +84,14 @@ func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, erro
|
||||
if ask.Build != nil {
|
||||
job := *ask.Build
|
||||
job.Fetch = withPackages(job.Fetch, ask.Packages)
|
||||
return steps.build(ctx, job, ask.Version)
|
||||
out, err := steps.build(ctx, job, ask.Version)
|
||||
var dec *Declining
|
||||
if err != nil && !sdkstep.IsPassing(err) && !errors.As(err, &dec) && ctx.Err() == nil {
|
||||
// A failed build keeps its error text, as the build module bounded it, as the ask's output object, header and
|
||||
// text as a test step's, so the verb output answers why after the fact (design 47 H2 H-A-482 and its addendum (a)).
|
||||
steps.keepRunOutput(ask.ID, err.Error(), nil)
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
if ask.Step != nil || ask.Install != nil {
|
||||
return steps.runStep(ctx, ask)
|
||||
|
||||
@@ -117,6 +117,23 @@ imports {"module":"modules/m","workspace":"/w/1"}`
|
||||
if !slices.Equal(answer.Imports, []string{"git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/health"}) {
|
||||
t.Errorf("the answer's imports are %v", answer.Imports)
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-482 addendum (a): a failed build keeps its error text, as the build module bounded it, as the ask's
|
||||
// output object, which the verb output answers.
|
||||
outputs := fakeOutputs{m: map[string][]byte{}}
|
||||
steps.Outputs, steps.Node = outputs, "ace"
|
||||
steps.Build = func(context.Context, map[string]any) (any, error) {
|
||||
return nil, errors.New("esbuild for code: status 1 | ✘ [ERROR] Could not resolve \"./missing\" … 812 bytes left out … main.js:1:1)")
|
||||
}
|
||||
failed := ask(sdkstep.BuildJob{Module: "m", Path: "modules/m", Fetch: &sdkstep.Fetch{Repository: "novox/x", Commit: runCommit},
|
||||
Artifacts: []sdkstep.AskArtifact{{Artifact: "bundle"}}})
|
||||
if _, err := RunAsk(context.Background(), steps, failed); err == nil {
|
||||
t.Fatal("a failed build answered no error")
|
||||
}
|
||||
got2, err := readOutput(outputs, "ace", failed.ID, nil)
|
||||
if err != nil || !strings.Contains(got2.Output, `bundle: esbuild for code: status 1 | ✘ [ERROR] Could not resolve "./missing"`) || got2.Size != len(got2.Output) {
|
||||
t.Errorf("the failed build's output = %+v, %v; want its error text kept", got2, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build ask for another toolchain is declined (ADR 0584 clause 2), so the node sits out and the asker puts it again.
|
||||
|
||||
@@ -125,7 +125,8 @@ func EndedHere(node string, keys func() ([]string, error), get func(key string)
|
||||
// readOutput answers the verb output for ask (design 47 H2 §791, H-A-412): from the object `<node>.<ask>`, the text's last
|
||||
// OutputTail bytes, its size and failing tests from the header; with no object, no longer kept when this node's steps key
|
||||
// shows the ask ended here more than OutputsMaxAge ago, and refused otherwise, naming why (an ask never run here, or one
|
||||
// whose output this node did not keep).
|
||||
// that kept no output). An empty object is no object (H-A-482): the runtime's object read answers an absent object as
|
||||
// empty data without an error, which is not a header fault.
|
||||
func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.Time, bool, error)) (sdkstep.OutputAnswer, error) {
|
||||
if o == nil {
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("this module keeps no %s store", OutputsStore)
|
||||
@@ -135,11 +136,14 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.
|
||||
return sdkstep.OutputAnswer{}, err
|
||||
}
|
||||
got, gerr := o.Get(name)
|
||||
if gerr == nil {
|
||||
if gerr == nil && len(got.Data) > 0 {
|
||||
return outputOf(ask, got.Data)
|
||||
}
|
||||
if gerr == nil {
|
||||
gerr = fmt.Errorf("the object %s in the %s store is empty", name, OutputsStore)
|
||||
}
|
||||
if ended == nil {
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the output %s in the %s store: %w", name, OutputsStore, gerr)
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s kept no output: %w", ask, gerr)
|
||||
}
|
||||
at, here, err := ended(ask)
|
||||
switch {
|
||||
@@ -150,7 +154,7 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.
|
||||
case !at.IsZero() && now().Sub(at) > OutputsMaxAge:
|
||||
return sdkstep.OutputAnswer{Ask: ask, NoLongerKept: true}, nil
|
||||
default:
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and its output is not in the %s store: %w", ask, node, OutputsStore, gerr)
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and the ask kept no output: %w", ask, node, gerr)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -64,7 +64,9 @@ func TestOutputsKeptAreRead(t *testing.T) {
|
||||
return raw
|
||||
}
|
||||
steps := map[string]json.RawMessage{"mesh-delivery-pipeline.ace.a2": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a3": step(at.Add(-time.Hour)),
|
||||
"mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour))}
|
||||
"mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a6": step(at.Add(-time.Hour))}
|
||||
// H-A-482: the runtime's object read answers an absent object as empty data, no error; that is no output, not a header fault.
|
||||
out.m["ace.a6"] = []byte{}
|
||||
keys := func() ([]string, error) {
|
||||
return slices.Collect(func(y func(string) bool) {
|
||||
for k := range steps {
|
||||
@@ -82,7 +84,8 @@ func TestOutputsKeptAreRead(t *testing.T) {
|
||||
}{
|
||||
{ask: "a1", want: sdkstep.OutputAnswer{Ask: "a1", Output: long[len(long)-OutputTail:], Size: len(long), Failing: failing}},
|
||||
{ask: "a2", want: sdkstep.OutputAnswer{Ask: "a2", NoLongerKept: true}},
|
||||
{ask: "a3", fail: "ran on ace, and its output is not in the outputs store"},
|
||||
{ask: "a3", fail: "ran on ace, and the ask kept no output"},
|
||||
{ask: "a6", fail: "ran on ace, and the ask kept no output: the object ace.a6 in the outputs store is empty"},
|
||||
{ask: "a4", fail: "never ran on ace"},
|
||||
{ask: "a5", fail: "never ran on ace"},
|
||||
{fail: "asked for no ask"},
|
||||
|
||||
@@ -267,8 +267,9 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
|
||||
return nil
|
||||
}
|
||||
|
||||
// keepRunOutput keeps a test step's level text as the run's output `<node>.<ask id>` in the outputs objects, with the
|
||||
// header of the failing tests and their words from the level's results (design 47 H2 §791, H-A-412 (1)). A holder that
|
||||
// keepRunOutput keeps a test step's level text, or a failed build's error text (H-A-482), as the run's output `<node>.<ask
|
||||
// id>` in the outputs objects, with the header of the failing tests and their words from the level's results (design 47
|
||||
// H2 §791, H-A-412 (1)). A holder that
|
||||
// keeps no outputs, or a write that fails, is said in the journal: the step's end stands on the results.
|
||||
func (s AskSteps) keepRunOutput(ask, text string, failed []sdkstep.Failure) {
|
||||
var failing []sdkstep.FailingTest
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -107,6 +108,30 @@ func (c creating) RaiseMaxBytes(stream string, to int64) (int64, bool, error) {
|
||||
return from, true, nil
|
||||
}
|
||||
|
||||
// AllowMsgTTL turns on a held stream's per-message time to live, its delete markers living markerTTL, when it is off, and
|
||||
// changes nothing else (design 47 H2 H-A-501, corrected): it reads the stream's configuration and updates with the rest as
|
||||
// it was. A stream with it on already is left as it is.
|
||||
func (c creating) AllowMsgTTL(stream string, markerTTL time.Duration) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
|
||||
defer cancel()
|
||||
s, err := c.js.Stream(ctx, stream)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
cfg := s.CachedInfo().Config
|
||||
if cfg.AllowMsgTTL {
|
||||
return false, nil
|
||||
}
|
||||
cfg.AllowMsgTTL = true
|
||||
if cfg.SubjectDeleteMarkerTTL == 0 {
|
||||
cfg.SubjectDeleteMarkerTTL = markerTTL
|
||||
}
|
||||
if _, err := c.js.UpdateStream(ctx, cfg); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// SourceFilter brings the filter subject of a held stream's source from `source` to `filter` and changes nothing else
|
||||
// (design 47 H H-A-145): it reads the stream's configuration and updates with the rest as it was, so its messages stay. A
|
||||
// stream with no source from `source`, or one already at `filter`, is left as it is.
|
||||
|
||||
@@ -5,13 +5,15 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The part's own membership names the stream running among those it hears, and hearRunning binds the part's durable on it:
|
||||
// The part's own membership names the stream running among those it hears, and hearRunning binds the part's durable on it
|
||||
// (and EVENTS, which the hub reads, H-A-472 seventh addendum):
|
||||
// the loop that opens the other streams leaves it alone, so no second reader takes messages of that one durable (design 47 G
|
||||
// §556, A-19; the whole-part integration test lost the running event to the second reader at random).
|
||||
func TestTheLoopOfOwnHearsLeavesTheStreamsADedicatedHearingOwns(t *testing.T) {
|
||||
hears := []membership.HeardIssued{{Stream: nodeengine.StreamRunning, Consumer: "a_mesh-grants"}, {Stream: "mod-c_feed", Consumer: "a_mesh-grants"}, {Stream: "opened_one"}}
|
||||
hears := []membership.HeardIssued{{Stream: nodeengine.StreamRunning, Consumer: "a_mesh-grants"}, {Stream: "mod-c_feed", Consumer: "a_mesh-grants"}, {Stream: "opened_one"}, {Stream: process.EventsStream, Consumer: "a_mesh-grants"}}
|
||||
got := hearsToOpen(hears, map[string]bool{"opened_one": true})
|
||||
if len(got) != 1 || got[0].Stream != "mod-c_feed" {
|
||||
t.Errorf("the streams opened are %+v, want mod-c_feed alone", got)
|
||||
|
||||
@@ -313,11 +313,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// hearsToOpen are the streams of the part's own membership no hearing has opened yet. A stream another hearing of the part
|
||||
// binds its durable on (running: hearRunning, which reads it whole at the start before it follows) is left to that hearing:
|
||||
// two readers of one durable each take the messages the other does not, and the event a reader of its own never hears is lost
|
||||
// to it (the integration test of the whole part met it as a state running that stayed empty, 2026-11-12).
|
||||
// to it (the integration test of the whole part met it as a state running that stayed empty, 2026-11-12). So is EVENTS, which
|
||||
// the membership names like any stream since it carries the consumer's spec (design 47 H2, H-A-472 seventh addendum): the
|
||||
// hub reads the part's durable there.
|
||||
func hearsToOpen(hears []membership.HeardIssued, opened map[string]bool) []membership.HeardIssued {
|
||||
var out []membership.HeardIssued
|
||||
for _, h := range hears {
|
||||
if !opened[h.Stream] && h.Stream != nodeengine.StreamRunning {
|
||||
if !opened[h.Stream] && h.Stream != nodeengine.StreamRunning && h.Stream != process.EventsStream {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,6 +202,19 @@ func (b *fakeBus) AddSubjects(stream string, add []string) error {
|
||||
// SourceFilter is the bus's update of a held stream's source filter: the fake holds no sources, so nothing changes.
|
||||
func (b *fakeBus) SourceFilter(string, string, string) (string, bool, error) { return "", false, nil }
|
||||
|
||||
// AllowMsgTTL is the bus's turning on of a held stream's per-message time to live, every other setting kept.
|
||||
func (b *fakeBus) AllowMsgTTL(stream string, _ time.Duration) (bool, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for _, s := range b.streams {
|
||||
if s["name"] == stream && s["allow_msg_ttl"] != true {
|
||||
s["allow_msg_ttl"] = true
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// RaiseMaxBytes is the bus's raise of a held stream's size cap: only to a larger cap, an unbounded stream and every other
|
||||
// setting kept.
|
||||
func (b *fakeBus) RaiseMaxBytes(stream string, to int64) (int64, bool, error) {
|
||||
@@ -719,10 +732,11 @@ func TestGrants(t *testing.T) {
|
||||
if n := strings.Count(string(src), "CreateStream"); (path == createOnly && n != 1) || (path != createOnly && n != 0) {
|
||||
t.Errorf("%s holds CreateStream %d times: only create.go creates, once", path, n)
|
||||
}
|
||||
// The three updates of a stream: adding subjects to a held one (the operator, 2026-11-05), raising its size cap
|
||||
// (2026-11-09) and bringing a kept stream's source filter to its definition's (design 47 H H-A-145), all in create.go.
|
||||
if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 3) || (path != createOnly && n != 0) {
|
||||
t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, three times, to add subjects, to raise a size cap and to bring a kept stream's source filter", path, n)
|
||||
// The four updates of a stream: adding subjects to a held one (the operator, 2026-11-05), raising its size cap
|
||||
// (2026-11-09), bringing a kept stream's source filter to its definition's (design 47 H H-A-145) and turning on a
|
||||
// key-ttl state's per-message time to live (design 47 H2 H-A-501, corrected), all in create.go.
|
||||
if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 4) || (path != createOnly && n != 0) {
|
||||
t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, four times, to add subjects, to raise a size cap, to bring a kept stream's source filter and to turn on per-message time to live", path, n)
|
||||
}
|
||||
// The one object bucket: the part's own, of the user list sealed to the bus node, in sealed.go (ADR 0547 clause 7).
|
||||
if path == sealedOnly {
|
||||
@@ -3157,6 +3171,24 @@ func TestGrants(t *testing.T) {
|
||||
if got := slices.Sorted(maps.Keys(b.own[grants.StateMemberships])); !slices.Equal(got, []string{"a.mod-x", "a.nats", "b.mod-y"}) {
|
||||
t.Errorf("memberships %v", got)
|
||||
}
|
||||
// Design 47 H2, H-A-472 ninth addendum: a module no longer assigned on its node has its membership withdrawn, so the
|
||||
// tool runner there removes its twins; a pass whose read failed withdraws nothing, and a bindings key is no membership.
|
||||
b.mu.Lock()
|
||||
b.own[grants.StateMemberships][grants.BoundKey("mod-y")] = []byte(`{}`)
|
||||
b.unassign("b", "mod-y")
|
||||
b.refused[grants.ReadHave] = true
|
||||
b.mu.Unlock()
|
||||
p.Compose()
|
||||
if _, _, found, _ := b.ownGet(grants.StateMemberships, "b.mod-y"); !found {
|
||||
t.Error("a pass whose read failed withdrew b.mod-y")
|
||||
}
|
||||
b.mu.Lock()
|
||||
delete(b.refused, grants.ReadHave)
|
||||
b.mu.Unlock()
|
||||
p.Compose()
|
||||
if got := slices.Sorted(maps.Keys(b.own[grants.StateMemberships])); !slices.Equal(got, []string{"a.mod-x", "a.nats", grants.BoundKey("mod-y")}) {
|
||||
t.Errorf("memberships once mod-y left b: %v", got)
|
||||
}
|
||||
for _, r := range grants.Reads {
|
||||
if strings.HasPrefix(r, "mesh-controller.") {
|
||||
t.Errorf("the part reads %s", r)
|
||||
@@ -3774,15 +3806,17 @@ func TestGrants(t *testing.T) {
|
||||
p := partOn(b)
|
||||
p.Consumers = b
|
||||
p.Compose()
|
||||
// The runner names no process of its own, so its consumers are its twins alone (H-A-472 second addendum).
|
||||
// The runner names no process of its own, so its consumers are its twins alone (H-A-472 second addendum), which the
|
||||
// runner creates itself from its membership's spec, granted their create by name (fifth to seventh addenda).
|
||||
for _, c := range []string{"INDEXER_VIEWS.a_node-tools_tools", "ASSIGNER_VIEWS.a_node-tools_tools"} {
|
||||
if !slices.Contains(b.consumerCreates, c) {
|
||||
t.Errorf("the tool runner's consumer %s was not made: %v", c, b.consumerCreates)
|
||||
if slices.Contains(b.consumerCreates, c) {
|
||||
t.Errorf("the part made the tool runner's twin %s: %v", c, b.consumerCreates)
|
||||
}
|
||||
}
|
||||
users := shadowOf(t, b)
|
||||
runner := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "a.node-tools" })]
|
||||
for _, s := range []string{"$JS.API.CONSUMER.MSG.NEXT.INDEXER_VIEWS.a_node-tools", "$KV.node-tools_findings.>"} {
|
||||
for _, s := range []string{"$JS.API.CONSUMER.MSG.NEXT.INDEXER_VIEWS.a_node-tools", "$KV.node-tools_findings.>",
|
||||
"$JS.API.CONSUMER.CREATE.INDEXER_VIEWS.a_node-tools_tools", "$JS.API.CONSUMER.DELETE.ASSIGNER_VIEWS.a_node-tools_tools"} {
|
||||
if !slices.ContainsFunc(runner.Publish, func(g string) bool { return buscompose.SubjectsOverlap(g, s) }) {
|
||||
t.Errorf("the tool runner may not publish %s: %v", s, runner.Publish)
|
||||
}
|
||||
@@ -4155,7 +4189,7 @@ func TestGrants(t *testing.T) {
|
||||
p.Consumers = b
|
||||
b.consumers = map[string][]map[string]any{"EVENTS": {{"name": "b_other", "filters": []string{"x"}, "max_deliver": 5}}}
|
||||
res := p.Compose()
|
||||
if res.Consumers == nil || !slices.Equal(res.Consumers.Created, []string{"EVENTS.b_mod-c", "EVENTS.b_mod-c_tools"}) {
|
||||
if res.Consumers == nil || !slices.Equal(res.Consumers.Created, []string{"EVENTS.b_mod-c"}) {
|
||||
t.Fatalf("with no lease key: %+v", res.Consumers)
|
||||
}
|
||||
var made map[string]any
|
||||
@@ -4168,7 +4202,7 @@ func TestGrants(t *testing.T) {
|
||||
t.Errorf("b_mod-c was made as %v, not as the controller makes it", made)
|
||||
}
|
||||
res = p.Compose()
|
||||
if len(res.Consumers.Created) != 0 || len(res.Consumers.Missing) != 0 || len(b.consumerCreates) != 2 {
|
||||
if len(res.Consumers.Created) != 0 || len(res.Consumers.Missing) != 0 || len(b.consumerCreates) != 1 {
|
||||
t.Errorf("a consumer the bus holds was asked again: %+v, %v", res.Consumers, b.consumerCreates)
|
||||
}
|
||||
// A held consumer that differs is never changed by a pass: it is raised as a condition naming it, and brought to
|
||||
@@ -4210,7 +4244,8 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
// 1126 (the self-check's bus-consumer probe ended): a consumer the part owns past the bound behind its stream's
|
||||
// head at two passes running raises mesh-grants.consumer-behind naming it; at the bound or under, it is cleared.
|
||||
made["pending"] = grants.BehindBound + 1
|
||||
// Read once, no pull now, far behind: behind (H-A-472 eleventh addendum), never an orphan.
|
||||
made["pending"], made["last_delivered"] = grants.BehindBound+1, "2026-10-20T21:00:00Z"
|
||||
if res = p.Compose(); len(res.Consumers.Behind) != 0 {
|
||||
t.Errorf("one look at a burst was said far behind: %+v", res.Consumers)
|
||||
}
|
||||
@@ -4233,12 +4268,9 @@ func TestGrants(t *testing.T) {
|
||||
// Design 47 H2, H-A-472 (2): a durable far behind with no pulls waiting, whose twin on the stream (b_mod-c_tools) has
|
||||
// pulls waiting and delivered within the day, is an orphan: left out of the behind check, named once in
|
||||
// mesh-grants.durable-orphaned, and cleared when it is gone; with its twin not current, it is behind again.
|
||||
var twin map[string]any
|
||||
for _, c := range b.consumers["EVENTS"] {
|
||||
if c["name"] == "b_mod-c_tools" {
|
||||
twin = c
|
||||
}
|
||||
}
|
||||
// The twin is the tool runner's, made by it from the membership's spec (the fifth addendum), never by the part.
|
||||
twin := map[string]any{"name": "b_mod-c_tools", "filters": []string{"mesh.mod.mod-y.event.done"}, "max_deliver": 5}
|
||||
b.consumers["EVENTS"] = append(b.consumers["EVENTS"], twin)
|
||||
twin["waiting_pulls"], twin["last_delivered"] = 1, "2026-10-25T21:00:00.5Z"
|
||||
made["pending"] = grants.BehindBound + 1
|
||||
p.Compose()
|
||||
@@ -4282,13 +4314,78 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
made["pending"] = grants.BehindBound
|
||||
p.Compose()
|
||||
// H-A-472 second addendum: a module with an account of its own but no process resource with no bus of its own has
|
||||
// no program reading b_<module>, so only the tool runner's twin is made.
|
||||
// H-A-472 second and fifth addenda: a module with an account of its own but no process resource with no bus of its own
|
||||
// has no program reading b_<module>, and the tool runner makes its own twin, so the part makes neither.
|
||||
b.manifestOf("mod-t", `{"module":"mod-t","version":"1","own-secrets":{"broker":"/var/lib/t/broker"},"consumes":["mod-y.done"]}`)
|
||||
b.assign("b", "mod-t")
|
||||
if res = p.Compose(); !slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t_tools") || slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t") {
|
||||
if res = p.Compose(); slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t_tools") || slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t") {
|
||||
t.Errorf("a module with no program of its own: created %v", res.Consumers.Created)
|
||||
}
|
||||
// The tenth and eleventh addenda: every held durable no pass composes is still judged, both twins of a module.
|
||||
named := func(list []string, key string) bool {
|
||||
return slices.ContainsFunc(list, func(s string) bool { return s == key || strings.HasPrefix(s, key+":") })
|
||||
}
|
||||
lastOrphans := func() string {
|
||||
if len(orphaned) == 0 || orphaned[len(orphaned)-1].State != occ.StateRaised {
|
||||
return ""
|
||||
}
|
||||
return orphaned[len(orphaned)-1].Explanation
|
||||
}
|
||||
// b_mod-c_tools, no pull waiting while the program's b_mod-c is current: an orphan by (2).
|
||||
twin["waiting_pulls"], twin["pending"] = 0, grants.BehindBound+1
|
||||
made["waiting_pulls"], made["last_delivered"], made["pending"] = 1, "2026-10-25T21:00:00.5Z", 0
|
||||
// b_mod-t_tools, never read (no pull, no delivery ever), mod-t having no program: an orphan, never behind.
|
||||
tTools := map[string]any{"name": "b_mod-t_tools", "filters": []string{"mesh.mod.mod-y.event.done"}, "max_deliver": 5,
|
||||
"pending": grants.BehindBound + 1}
|
||||
b.consumers["EVENTS"] = append(b.consumers["EVENTS"], tTools)
|
||||
p.Compose()
|
||||
res = p.Compose()
|
||||
if named(res.Consumers.Behind, "EVENTS.b_mod-t_tools") || named(res.Consumers.Behind, "EVENTS.b_mod-c_tools") ||
|
||||
!strings.Contains(lastOrphans(), "EVENTS.b_mod-c_tools") || !strings.Contains(lastOrphans(), "EVENTS.b_mod-t_tools") {
|
||||
t.Errorf("the held twins: behind %v, orphaned said %q", res.Consumers.Behind, lastOrphans())
|
||||
}
|
||||
// Read once, no pull now, far behind, with no current twin: behind.
|
||||
tTools["last_delivered"] = "2026-10-20T21:00:00Z"
|
||||
p.Compose()
|
||||
if res = p.Compose(); !named(res.Consumers.Behind, "EVENTS.b_mod-t_tools") || strings.Contains(lastOrphans(), "EVENTS.b_mod-t_tools") {
|
||||
t.Errorf("a twin read once and lagging: behind %v, orphaned said %q", res.Consumers.Behind, lastOrphans())
|
||||
}
|
||||
// The plain b_mod-t, which no pass composes (mod-t's program left), far behind with no pull beside a current
|
||||
// b_mod-t_tools: an orphan by (2).
|
||||
tTools["waiting_pulls"], tTools["last_delivered"], tTools["pending"] = 1, "2026-10-25T21:00:00.5Z", 0
|
||||
b.consumers["EVENTS"] = append(b.consumers["EVENTS"], map[string]any{"name": "b_mod-t", "filters": []string{"mesh.mod.mod-y.event.done"},
|
||||
"max_deliver": 5, "pending": grants.BehindBound + 1, "last_delivered": "2026-10-20T21:00:00Z"})
|
||||
p.Compose()
|
||||
if res = p.Compose(); named(res.Consumers.Behind, "EVENTS.b_mod-t") || !strings.Contains(lastOrphans(), "EVENTS.b_mod-t,") &&
|
||||
!strings.HasSuffix(lastOrphans(), "EVENTS.b_mod-t") {
|
||||
t.Errorf("the plain durable of a module whose program left: behind %v, orphaned said %q", res.Consumers.Behind, lastOrphans())
|
||||
}
|
||||
// A stream only twins are held on (no pass composes a consumer there): mod-s keeps the stream feed, which the
|
||||
// program-less mod-u hears; its never-read twin there is judged and named.
|
||||
b.manifestOf("mod-s", `{"module":"mod-s","version":"1","emits":["fed"],"streams":[{"name":"feed","events":["fed"],"max-age-days":2,"max-mib":5}]}`)
|
||||
b.manifestOf("mod-u", `{"module":"mod-u","version":"1","own-secrets":{"broker":"/var/lib/u/broker"},"consumes":["mod-s.fed"]}`)
|
||||
b.assign("b", "mod-s")
|
||||
b.assign("b", "mod-u")
|
||||
p.Compose()
|
||||
var feed string
|
||||
raw, _, _, _ := b.ownGet(grants.StateMemberships, "b.mod-u")
|
||||
if bare, err := membership.BareGrantsMembership(raw); err == nil {
|
||||
var m membership.Membership
|
||||
_ = json.Unmarshal(bare, &m)
|
||||
for _, h := range m.Hears {
|
||||
if h.Stream != "EVENTS" {
|
||||
feed = h.Stream
|
||||
}
|
||||
}
|
||||
}
|
||||
if feed == "" {
|
||||
t.Fatalf("mod-u's membership hears no stream of mod-s: %s", raw)
|
||||
}
|
||||
b.consumers[feed] = []map[string]any{{"name": "b_mod-u_tools", "filters": []string{"x"}, "max_deliver": 5, "pending": grants.BehindBound + 1}}
|
||||
p.Compose()
|
||||
if res = p.Compose(); !strings.Contains(lastOrphans(), feed+".b_mod-u_tools") {
|
||||
t.Errorf("a twin on a stream no pass composes a consumer on: orphaned said %q (unread %v)", lastOrphans(), res.Consumers.Unread)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/AC13 (novox/hq ADR 0561 §7): after take-over a list in which more than one loaded user's hash changes, with no
|
||||
|
||||
@@ -640,9 +640,9 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
|
||||
MayPublish(perms, "$JS.API.CONSUMER.DELETE.EVENTS.a_m") || MayPublish(perms, "$JS.API.CONSUMER.CREATE.EVENTS.a_m.mesh.mod.x.event.happened") {
|
||||
t.Errorf("the grants part creates consumers as %v", u.CreatesConsumers)
|
||||
}
|
||||
// Design 47 H2, H-A-472 second addendum: the program's consumer only for a module with a program of its own; the
|
||||
// tool runner's twin for every consuming module.
|
||||
for durable, want := range map[string]bool{"a_m": true, "a_m_tools": true, "a_tooled": false, "a_tooled_tools": true} {
|
||||
// Design 47 H2, H-A-472 second and fifth addenda: the program's consumer only for a module with a program of its own;
|
||||
// the tool runner's twin never, which the runner creates itself from the membership's spec.
|
||||
for durable, want := range map[string]bool{"a_m": true, "a_m_tools": false, "a_tooled": false, "a_tooled_tools": false} {
|
||||
if MayPublish(perms, "$JS.API.CONSUMER.CREATE.EVENTS."+durable) != want {
|
||||
t.Errorf("the grants part creates EVENTS.%s: %v, want %v (%v)", durable, !want, want, u.CreatesConsumers)
|
||||
}
|
||||
@@ -1546,7 +1546,9 @@ func theToolRunnerHearsEveryEvent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
for _, s := range p.Publish {
|
||||
if strings.Contains(s, "LEDGER") || strings.HasPrefix(s, "$JS.ACK.") {
|
||||
// A module's own takeins durable on the shared events stream is no restricted stream (H-A-500 (b)).
|
||||
takeIns := strings.HasPrefix(s, "$JS.ACK."+EventsStream+".") && strings.HasSuffix(s, TakeInsSuffix+".>")
|
||||
if strings.Contains(s, "LEDGER") || (strings.HasPrefix(s, "$JS.ACK.") && !takeIns) {
|
||||
t.Errorf("the tool runner's runtime may %s on the restricted stream", s)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,6 +163,14 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
||||
}, true
|
||||
}
|
||||
|
||||
// carriedPrincipal is a module the tool runner carries as the consumers it reads are composed from: the one form ConsumersOf
|
||||
// composes a carried module's consumers and HeardIssuedFor their spec from, so the two cannot differ.
|
||||
func carriedPrincipal(node string, d Declared) Principal {
|
||||
return Principal{Kind: KindModule, Node: node, Module: d.Module, HasProgram: d.HasProgram, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||
Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues}
|
||||
}
|
||||
|
||||
// ModuleConsumer is one module's durable consumer, with the module and node it is for.
|
||||
type ModuleConsumer struct {
|
||||
Node, Module string
|
||||
@@ -191,8 +199,8 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
}
|
||||
seen[key] = true
|
||||
if needed {
|
||||
// Each with the tool runner's twin beside it (A-397), the program's only for a module with a program of its
|
||||
// own (H-A-472 second addendum).
|
||||
// The program's only, for a module with a program of its own (H-A-472 second addendum); the tool runner's twin is
|
||||
// the runner's to make from the membership's spec (H-A-472 fifth to seventh addenda).
|
||||
for _, tc := range twinsOf(c, p.HasProgram) {
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: tc})
|
||||
}
|
||||
@@ -244,9 +252,7 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
// 0436 §1, §2, §6), the same reason its ordinary consumer is (ADR 0198): mesh-mail has no
|
||||
// account of its own (NoAccount) and is carried here, but its process does, and the consumer
|
||||
// it reads through must exist and survive PruneHeard whether or not the module itself speaks.
|
||||
add(Principal{Kind: KindModule, Node: p.Node, Module: d.Module, HasProgram: d.HasProgram, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||
Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues})
|
||||
add(carriedPrincipal(p.Node, d))
|
||||
}
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
@@ -253,40 +254,70 @@ func heardConsumer(p Principal, h Heard) Consumer {
|
||||
// under a name of its own, since only a work queue shares one (lead answer A-397; mesh-tools bus.ToolsConsumerOf).
|
||||
const ToolsSuffix = "_tools"
|
||||
|
||||
// TakeInsSuffix ends the name of a module's own second durable on the shared events stream, `<node>_<module>_takeins`,
|
||||
// which the module makes itself and hears take-in.taken on alone, so a take-in never waits behind its other events (design
|
||||
// 47 H2 H-A-500 (b) and its addenda): every module principal is granted it, none special-cased.
|
||||
const TakeInsSuffix = "_takeins"
|
||||
|
||||
// takeInsGrants are what a module publishes to make, ask about, pull from and acknowledge its takeins durable on the shared
|
||||
// events stream (TakeInsSuffix): the create by its name alone, every filter in FilterSubjects.
|
||||
func takeInsGrants(durable string) []string {
|
||||
d := durable + TakeInsSuffix
|
||||
return []string{"$JS.API.CONSUMER.CREATE." + EventsStream + "." + d, "$JS.API.CONSUMER.INFO." + EventsStream + "." + d,
|
||||
"$JS.API.CONSUMER.MSG.NEXT." + EventsStream + "." + d, "$JS.ACK." + EventsStream + "." + d + ".>"}
|
||||
}
|
||||
|
||||
// readerGrants are what a reader publishes to ask about, pull from and acknowledge a durable consumer on a stream: for the
|
||||
// module's program's name and for its tool runner's twin (A-397).
|
||||
// module's program's name and for its tool runner's twin (A-397); and the create and delete of the twin by its name alone,
|
||||
// which the tool runner makes from the membership's spec and removes when the module leaves the node (design 47 H2, H-A-472
|
||||
// fifth to seventh addenda). Every filter goes in FilterSubjects, so the create is on the name, never on a filter.
|
||||
func readerGrants(stream, durable string) []string {
|
||||
var out []string
|
||||
for _, d := range []string{durable, durable + ToolsSuffix} {
|
||||
out = append(out, "$JS.API.CONSUMER.INFO."+stream+"."+d, "$JS.API.CONSUMER.MSG.NEXT."+stream+"."+d, "$JS.ACK."+stream+"."+d+".>")
|
||||
}
|
||||
return out
|
||||
twin := durable + ToolsSuffix
|
||||
return append(out, "$JS.API.CONSUMER.CREATE."+stream+"."+twin, "$JS.API.CONSUMER.DELETE."+stream+"."+twin)
|
||||
}
|
||||
|
||||
// withToolsTwin is a module's consumer and, beside it, the tool runner's twin of it: the same filters under its own name (A-397).
|
||||
func withToolsTwin(c Consumer) []Consumer {
|
||||
// toolsTwin is the tool runner's twin of a module's consumer c: the same filters under its own name (A-397), the events
|
||||
// given up on and delivered again to one consumer alone named by the twin itself.
|
||||
func toolsTwin(c Consumer) Consumer {
|
||||
twin := c
|
||||
twin.Name += ToolsSuffix
|
||||
twin.Filters = append([]string(nil), c.Filters...)
|
||||
// The events given up on and delivered again to one consumer alone are named by it: the twin's are its own.
|
||||
for i, f := range twin.Filters {
|
||||
if f == AgainFilter(c.Name) {
|
||||
twin.Filters[i] = AgainFilter(twin.Name)
|
||||
}
|
||||
}
|
||||
twin.Why = c.Why + "; the tool runner's own, beside the program's (lead answer A-397)"
|
||||
return []Consumer{c, twin}
|
||||
return twin
|
||||
}
|
||||
|
||||
// twinsOf are the consumers composed of a module's consumer c (design 47 H2, H-A-472 addendum and second addendum): the
|
||||
// tool runner's twin <name>_tools for every consuming module, and c itself, the program's, only for a module with a
|
||||
// program of its own (Declared.HasProgram), so no pass makes a durable nobody reads.
|
||||
// twinsOf are the consumers the grants part composes of a module's consumer c (design 47 H2, H-A-472 addendum, second and
|
||||
// fifth addenda): c itself, the program's, only for a module with a program of its own (Declared.HasProgram), and never the
|
||||
// tool runner's twin <name>_tools, which the runner creates from the membership's spec at a child's first subscribe.
|
||||
func twinsOf(c Consumer, hasProgram bool) []Consumer {
|
||||
both := withToolsTwin(c)
|
||||
if hasProgram {
|
||||
return both
|
||||
return []Consumer{c}
|
||||
}
|
||||
return both[1:]
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardSpec is a heard entry of consumer c's twin: its stream, the module's consumer name the twin is named from, and the
|
||||
// twin's whole spec (H-A-472 seventh addendum).
|
||||
func heardSpec(c Consumer) HeardIssued {
|
||||
twin := toolsTwin(c)
|
||||
policy := membership.DeliverAll
|
||||
switch {
|
||||
case twin.LastPerSubject:
|
||||
policy = membership.DeliverLastPerSubject
|
||||
case twin.FromNow:
|
||||
policy = membership.DeliverFromNow
|
||||
}
|
||||
return HeardIssued{Stream: c.Stream, Consumer: c.Name, Filters: twin.Filters, AckWaitSeconds: twin.AckWaitSeconds,
|
||||
MaxDeliver: twin.MaxDeliver, MaxAckPending: twin.MaxAckPending, DeliverPolicy: policy}
|
||||
}
|
||||
|
||||
// heardGrants are what a principal publishes to read and acknowledge the consumers it hears module streams
|
||||
@@ -345,15 +376,21 @@ func streamsIssuedFor(d Declared) []StreamIssued {
|
||||
return out
|
||||
}
|
||||
|
||||
// HeardIssuedFor is every module stream an assignment hears, as its membership names it.
|
||||
// HeardIssuedFor is every stream an assignment hears, as its membership names it: EVENTS like any stream, each entry with
|
||||
// its consumer's whole spec as the tool runner's twin of it is composed (design 47 H2, H-A-472 seventh addendum), so the
|
||||
// runner computes no name and no filter and the grants part stays the one composer of filters.
|
||||
func HeardIssuedFor(node string, d Declared) []HeardIssued {
|
||||
var out []HeardIssued
|
||||
p := carriedPrincipal(node, d)
|
||||
if c, needed := ConsumerFor(p); needed {
|
||||
out = append(out, heardSpec(c))
|
||||
}
|
||||
// A restricted stream's bind word only for a module of its own account (novox/hq ADR 0376 §1).
|
||||
for _, h := range ComposedHears(d) {
|
||||
for _, h := range p.Hears {
|
||||
if len(h.Filters) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, HeardIssued{Stream: h.Stream, Consumer: node + "_" + d.Module})
|
||||
out = append(out, heardSpec(heardConsumer(p, h)))
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Stream < out[j].Stream })
|
||||
return out
|
||||
|
||||
@@ -1159,6 +1159,8 @@ func permissionsFor(p Principal) (Permissions, error) {
|
||||
// which asks for these permissions to build the consumer and would ask forever. A
|
||||
// subject for a consumer that turns out not to exist grants nothing anybody can use.
|
||||
pub = append(pub, readerGrants(consumerStream(p), consumerDurable(p))...)
|
||||
// 2d. Its own takeins durable on the shared events stream, made and heard by the module (H-A-500 (b)).
|
||||
pub = append(pub, takeInsGrants(consumerDurable(p))...)
|
||||
|
||||
// 3. Seats it holds: full participation. But mesh-broker's acting verbs, which nobody serves on the bus (busstate.go).
|
||||
for _, s := range p.Holds {
|
||||
@@ -1326,6 +1328,8 @@ func permissionsFor(p Principal) (Permissions, error) {
|
||||
// And the module streams it hears, each through its own consumer there (novox/hq ADR 0363 §8) — never
|
||||
// a restricted one, which only a module on an account of its own hears (ADR 0376 §1).
|
||||
pub = append(pub, heardGrants(own, own.Hears)...)
|
||||
// Its takeins durable on the shared events stream, as a module on an account of its own has (H-A-500 (b)).
|
||||
pub = append(pub, takeInsGrants(consumerDurable(own))...)
|
||||
if _, consumes := ConsumerFor(own); !consumes {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ accounts {
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "one.mesh-delivery-watchman", password: "$2a$11$wwwwwwwwwwwwwwwwwwwwww", permissions: {
|
||||
publish: { allow: ["$JS.ACK.DEAD_LETTER_NOTICES.mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman_tools.>", "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.DIRECT.GET.PROV_APPLIED.>", "$JS.API.STREAM.INFO.DEAD_LETTERS", "$JS.API.STREAM.INFO.DEAD_LETTER_NOTICES", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.INFO.PROV_APPLIED", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS", "$JS.API.STREAM.MSG.GET.PROV_APPLIED", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.bus.connz", "mesh.events.dead.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~mesh-delivery-watchman", "mesh.seat.*.call.*.one~mesh-delivery-watchman", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.DEAD_LETTER_NOTICES.mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman_takeins.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman_tools.>", "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.CREATE.EVENTS.one_mesh-delivery-watchman_takeins", "$JS.API.CONSUMER.CREATE.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.CONSUMER.DELETE.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman_takeins", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman_takeins", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.DIRECT.GET.PROV_APPLIED.>", "$JS.API.STREAM.INFO.DEAD_LETTERS", "$JS.API.STREAM.INFO.DEAD_LETTER_NOTICES", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.INFO.PROV_APPLIED", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS", "$JS.API.STREAM.MSG.GET.PROV_APPLIED", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.bus.connz", "mesh.events.dead.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~mesh-delivery-watchman", "mesh.seat.*.call.*.one~mesh-delivery-watchman", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$SRV.INFO", "$SRV.INFO.mesh-delivery-watchman", "$SRV.INFO.mesh-delivery-watchman.>", "$SRV.PING", "$SRV.PING.mesh-delivery-watchman", "$SRV.PING.mesh-delivery-watchman.>", "$SRV.STATS", "$SRV.STATS.mesh-delivery-watchman", "$SRV.STATS.mesh-delivery-watchman.>", "_INBOX.one.mesh-delivery-watchman.>", "mesh.bus.connect", "mesh.bus.disconnect", "mesh.control.*.tools-alive", "mesh.events.dead.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.mesh-delivery-watchman.call.>", "mesh.mod.mesh-delivery-watchman.tool.>", "mesh.mod.nats.event.bus.connection-closed", "mesh.mod.nats.event.bus.slow-consumer", "mesh.mod.power.event.*", "mesh.seat.mesh-self-check.event.checked", "prov.applied.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
@@ -47,17 +47,17 @@ accounts {
|
||||
subscribe: { allow: ["_INBOX.one.nats.>"] }
|
||||
} }
|
||||
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.EVENTS.one_telegram_tools.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.telegram", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.telegram", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~telegram", "mesh.seat.*.call.*.one~telegram", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.EVENTS.one_telegram_takeins.>", "$JS.ACK.EVENTS.one_telegram_tools.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.CREATE.EVENTS.one_telegram_takeins", "$JS.API.CONSUMER.CREATE.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.telegram", "$JS.API.CONSUMER.DELETE.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram_takeins", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram_takeins", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.telegram", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~telegram", "mesh.seat.*.call.*.one~telegram", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$JS.EVENT.ADVISORY.CONSUMER.CREATED.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.mod.telegram.call.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.ACK.EVENTS.two_audit_tools.>", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.audit", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.INFO.EVENTS.two_audit_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.audit", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~audit", "mesh.seat.*.call.*.two~audit", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.ACK.EVENTS.two_audit_takeins.>", "$JS.ACK.EVENTS.two_audit_tools.>", "$JS.API.CONSUMER.CREATE.EVENTS.two_audit_takeins", "$JS.API.CONSUMER.CREATE.EVENTS.two_audit_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.audit", "$JS.API.CONSUMER.DELETE.EVENTS.two_audit_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.INFO.EVENTS.two_audit_takeins", "$JS.API.CONSUMER.INFO.EVENTS.two_audit_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit_takeins", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.audit", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~audit", "mesh.seat.*.call.*.two~audit", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.mod.audit.call.>", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.ACK.EVENTS.two_shop_tools.>", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.shop", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.INFO.EVENTS.two_shop_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.shop", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.mod.shop.event.order.placed", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~shop", "mesh.seat.*.call.*.two~shop", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.accept.send"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.ACK.EVENTS.two_shop_takeins.>", "$JS.ACK.EVENTS.two_shop_tools.>", "$JS.API.CONSUMER.CREATE.EVENTS.two_shop_takeins", "$JS.API.CONSUMER.CREATE.EVENTS.two_shop_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.shop", "$JS.API.CONSUMER.DELETE.EVENTS.two_shop_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.INFO.EVENTS.two_shop_takeins", "$JS.API.CONSUMER.INFO.EVENTS.two_shop_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop_takeins", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.shop", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.mod.shop.event.order.placed", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~shop", "mesh.seat.*.call.*.two~shop", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.accept.send"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.mod.shop.call.>", "mesh.mod.shop.tool.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -3,20 +3,103 @@ package buscompose
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
)
|
||||
|
||||
// A-397: beside each module's durable consumer the tool runner's twin, with the same filters under its own name, and the
|
||||
// events delivered again to one consumer named by the twin itself.
|
||||
func TestTheRunnersTwinBesideTheModulesConsumer(t *testing.T) {
|
||||
c := Consumer{Name: "ace_mod", Stream: "EVENTS", Filters: []string{"mesh.mod.x.event.y", AgainFilter("ace_mod")}}
|
||||
got := withToolsTwin(c)
|
||||
if len(got) != 2 || got[0].Name != "ace_mod" || got[1].Name != "ace_mod_tools" {
|
||||
t.Fatalf("consumers = %+v, want ace_mod and ace_mod_tools", got)
|
||||
// Design 47 H2, H-A-472 fifth to seventh addenda: the grants part composes no tool runner's twin; it writes the twin's whole
|
||||
// spec in the membership's heard entries, EVENTS like any stream, and grants the module's reader the create and delete of
|
||||
// its own twin by name.
|
||||
func TestTheRunnersTwinIsTheMembershipsSpec(t *testing.T) {
|
||||
c := Consumer{Name: "ace_mod", Stream: "EVENTS", Filters: []string{"mesh.mod.x.event.y", AgainFilter("ace_mod")}, AckWaitSeconds: 30,
|
||||
MaxDeliver: 5, FromNow: true}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
hasProgram bool
|
||||
want []string
|
||||
}{
|
||||
{"a module with a program of its own: the program's alone", true, []string{"ace_mod"}},
|
||||
{"a module the tool runner carries: none", false, nil},
|
||||
} {
|
||||
var got []string
|
||||
for _, tc := range twinsOf(c, row.hasProgram) {
|
||||
got = append(got, tc.Name)
|
||||
}
|
||||
if !slices.Equal(got, row.want) {
|
||||
t.Errorf("%s: consumers composed = %v, want %v", row.name, got, row.want)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got[1].Filters, []string{"mesh.mod.x.event.y", AgainFilter("ace_mod_tools")}) {
|
||||
t.Errorf("twin filters = %v", got[1].Filters)
|
||||
|
||||
d := Declared{Module: "mod", Consumes: []string{"x.happened"},
|
||||
Hears: []Heard{{Stream: "feed_one", Emitter: "feed", Filters: []string{"mesh.stream.feed.event.a"}},
|
||||
{Stream: "kept_one", Emitter: "keeper", Filters: []string{"mesh.stream.keeper.event.k"}, LastPerSubject: true}}}
|
||||
hears := HeardIssuedFor("ace", d)
|
||||
byStream := map[string]membership.HeardIssued{}
|
||||
for _, h := range hears {
|
||||
byStream[h.Stream] = h
|
||||
if !h.HasSpec() || h.Consumer != "ace_mod" || h.AckWaitSeconds != 30 || h.MaxDeliver != 5 {
|
||||
t.Errorf("heard entry %+v: want the twin's spec under the module's consumer ace_mod", h)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got[0].Filters, c.Filters) {
|
||||
t.Errorf("the module's filters changed: %v", got[0].Filters)
|
||||
for _, row := range []struct {
|
||||
stream, policy string
|
||||
filters []string
|
||||
}{
|
||||
{"EVENTS", membership.DeliverFromNow, []string{"mesh.mod.x.event.happened", AgainFilter("ace_mod_tools")}},
|
||||
{"feed_one", membership.DeliverFromNow, []string{"mesh.stream.feed.event.a"}},
|
||||
{"kept_one", membership.DeliverLastPerSubject, []string{"mesh.stream.keeper.event.k"}},
|
||||
} {
|
||||
h, ok := byStream[row.stream]
|
||||
if !ok {
|
||||
t.Errorf("no heard entry for %s in %+v", row.stream, hears)
|
||||
continue
|
||||
}
|
||||
if h.DeliverPolicy != row.policy || !slices.Equal(h.Filters, row.filters) {
|
||||
t.Errorf("%s: policy %q filters %v, want %q %v", row.stream, h.DeliverPolicy, h.Filters, row.policy, row.filters)
|
||||
}
|
||||
}
|
||||
|
||||
grants := readerGrants("EVENTS", "ace_mod")
|
||||
for _, row := range []struct {
|
||||
subject string
|
||||
want bool
|
||||
}{
|
||||
{"$JS.API.CONSUMER.CREATE.EVENTS.ace_mod_tools", true},
|
||||
{"$JS.API.CONSUMER.DELETE.EVENTS.ace_mod_tools", true},
|
||||
{"$JS.API.CONSUMER.INFO.EVENTS.ace_mod_tools", true},
|
||||
{"$JS.API.CONSUMER.CREATE.EVENTS.ace_mod", false},
|
||||
{"$JS.API.CONSUMER.DELETE.EVENTS.ace_mod", false},
|
||||
} {
|
||||
if slices.Contains(grants, row.subject) != row.want {
|
||||
t.Errorf("reader grants %s: %v, want %v (%v)", row.subject, !row.want, row.want, grants)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H2, H-A-500 (b) and its addenda: every module principal, on an account of its own or carried by a tool runner,
|
||||
// is granted the create, information, pull and acknowledgement of its own takeins durable on the shared events stream.
|
||||
func TestEveryModuleIsGrantedItsTakeInsDurable(t *testing.T) {
|
||||
takeIns := func(node, module string) []string {
|
||||
d := node + "_" + module + TakeInsSuffix
|
||||
return []string{"$JS.API.CONSUMER.CREATE.EVENTS." + d, "$JS.API.CONSUMER.INFO.EVENTS." + d,
|
||||
"$JS.API.CONSUMER.MSG.NEXT.EVENTS." + d, "$JS.ACK.EVENTS." + d + ".>"}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
p Principal
|
||||
module string
|
||||
}{
|
||||
{"a module's own account", Principal{Kind: KindModule, Node: "a", Module: "shop"}, "shop"},
|
||||
{"a module a tool runner carries", Principal{Kind: KindNodeTools, Node: "a", Module: "node-tools", Carries: []Declared{{Module: "shop"}}}, "shop"},
|
||||
} {
|
||||
perms, err := PermissionsFor(c.p)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.name, err)
|
||||
}
|
||||
for _, g := range takeIns("a", c.module) {
|
||||
if !slices.Contains(perms.Publish, g) {
|
||||
t.Errorf("%s is not granted %s", c.name, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,7 +90,8 @@ type Declared struct {
|
||||
NoAccount bool
|
||||
// HasProgram says the module runs a program of its own that reads its consumer <node>_<module>: an account of its own
|
||||
// (an own broker secret) and a process resource with no `bus` of its own (design 47 H2, H-A-472 second addendum).
|
||||
// Without it, only the tool runner's twin <node>_<module>_tools is composed, so the pass makes no orphan.
|
||||
// Without it, nothing is composed on its consumer's name, so the pass makes no orphan; the tool runner's twin
|
||||
// <node>_<module>_tools is the runner's to make from the membership's spec (H-A-472 fifth to seventh addenda).
|
||||
HasProgram bool
|
||||
// SnapshotsTheBus says the module holds mesh-broker — it is the bus — and so is the one module
|
||||
// granted the snapshot API, to copy the bus's streams for the night's backup (novox/hq ADR 0235).
|
||||
|
||||
@@ -30,6 +30,7 @@ func (c *askCreator) RaiseMaxBytes(string, int64) (int64, bool, error) {
|
||||
func (c *askCreator) SourceFilter(string, string, string) (string, bool, error) {
|
||||
return "", false, nil
|
||||
}
|
||||
func (c *askCreator) AllowMsgTTL(string, time.Duration) (bool, error) { return false, nil }
|
||||
|
||||
// A seat whose queues came with G (mesh-builds) gets its work-queue ask stream from the seat list, as the controller made one for
|
||||
// every seat before G; a seat that accepts nothing gets none, and a stream the bus holds is never made again (lead-47, 2026-11-15).
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -74,6 +75,10 @@ type Creator interface {
|
||||
// messages and every other setting, the stream not made again (design 47 H H-A-145). It answers the filter it held and
|
||||
// whether it changed; a stream with no source from `source` is left as it is.
|
||||
SourceFilter(stream, source, filter string) (from string, changed bool, err error)
|
||||
// AllowMsgTTL turns on per-message time to live on a stream the bus holds, its delete markers living markerTTL, when it
|
||||
// is off, and changes nothing else (design 47 H2 H-A-501, corrected): a state declared key-ttl whose bucket was made
|
||||
// before the declaration. It answers whether it turned it on; one on already is left as it is. It never turns it off.
|
||||
AllowMsgTTL(stream string, markerTTL time.Duration) (turned bool, err error)
|
||||
}
|
||||
|
||||
// Raise is one held stream's size cap raised to the manifest's, said in the event and the journal.
|
||||
@@ -126,6 +131,24 @@ type creations struct {
|
||||
// refused is, by stream, the refusal of the last addition of subjects (an overlap with another stream, say), so a
|
||||
// refusal that stands is said once and not at every pass; a success or a different refusal says again.
|
||||
refused map[string]string
|
||||
// ttl is, by stream, the manifest digest at which its per-message time to live was last compared with the manifest's
|
||||
// key-ttl, so the bus is asked once per manifest (a hand assert asks again).
|
||||
ttl map[string]string
|
||||
}
|
||||
|
||||
func (c *creations) ttlAt(stream string) string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.ttl[stream]
|
||||
}
|
||||
|
||||
func (c *creations) keepTTL(stream, digest string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.ttl == nil {
|
||||
c.ttl = map[string]string{}
|
||||
}
|
||||
c.ttl[stream] = digest
|
||||
}
|
||||
|
||||
// refusedAgain says whether this refusal of an addition to the stream was the last one said, and keeps it.
|
||||
@@ -265,6 +288,26 @@ func createMissing(c Creator, done *creations, a *Assertion, m catalogue.Manifes
|
||||
made = append(made, fmt.Sprintf("%s %s size cap raised from %d to %d bytes", it.Kind, it.Stream, from, it.Composed.MaxBytes))
|
||||
}
|
||||
}
|
||||
// A held bucket of a state declared key-ttl gets per-message time to live when it lacks it, add-only like the cap: a
|
||||
// bucket made before its state was declared key-ttl refuses every key written with a time to live (design 47 H2
|
||||
// H-A-501, corrected). Compared once per manifest.
|
||||
for i := range a.Items {
|
||||
it := &a.Items[i]
|
||||
cfg, ok := cfgs[it.Stream]
|
||||
if it.Held == nil || it.Kind != "bucket" || !ok || !cfg.AllowMsgTTL || (!force && done.ttlAt(it.Stream) == a.Manifest) {
|
||||
continue
|
||||
}
|
||||
turned, err := c.AllowMsgTTL(it.Stream, cfg.SubjectDeleteMarkerTTL)
|
||||
if err != nil {
|
||||
it.Created = ItemNotCreated + "per-message time to live not turned on: " + err.Error() + " (tried again at the next event)"
|
||||
continue
|
||||
}
|
||||
done.keepTTL(it.Stream, a.Manifest)
|
||||
if turned {
|
||||
it.Created = "per-message time to live turned on (its state is declared key-ttl)"
|
||||
made = append(made, it.Kind+" "+it.Stream+" per-message time to live turned on, its state declared key-ttl")
|
||||
}
|
||||
}
|
||||
if len(made) > 0 {
|
||||
a.Asserted = true
|
||||
}
|
||||
|
||||
@@ -118,7 +118,8 @@ func durableTwin(name string) string {
|
||||
return name + "_tools"
|
||||
}
|
||||
|
||||
// orphaned says whether a held durable is an orphan (design 47 H2, H-A-472 (2)): no pulls waiting on it, while its twin
|
||||
// orphaned says whether a held durable is an orphan (design 47 H2, H-A-472 (2), eleventh addendum): never read at all, or
|
||||
// no pulls waiting on it, while its twin
|
||||
// on the same stream is current, with pulls waiting and a delivery within the bound (the setting twin-current-hours,
|
||||
// the third addendum). A module that moved into the tool runner left its program's durable so, and one that runs its own
|
||||
// program again its tool runner's.
|
||||
@@ -126,6 +127,11 @@ func orphaned(h heldConsumer, held map[string]heldConsumer, now time.Time, withi
|
||||
if h.WaitingPulls > 0 {
|
||||
return false
|
||||
}
|
||||
// A durable nobody ever read, no pull waiting and no delivery ever, is an orphan whatever its twin's state; one that was
|
||||
// read and lags is behind (H-A-472 eleventh addendum).
|
||||
if neverDelivered(h) {
|
||||
return true
|
||||
}
|
||||
t, ok := held[durableTwin(h.Name)]
|
||||
if !ok || t.WaitingPulls == 0 {
|
||||
return false
|
||||
@@ -134,6 +140,15 @@ func orphaned(h heldConsumer, held map[string]heldConsumer, now time.Time, withi
|
||||
return err == nil && now.Sub(last) < within
|
||||
}
|
||||
|
||||
// neverDelivered says the bus never delivered from the durable: no time of a last delivery, or the zero time.
|
||||
func neverDelivered(h heldConsumer) bool {
|
||||
if h.LastDelivered == "" {
|
||||
return true
|
||||
}
|
||||
last, err := time.Parse(time.RFC3339Nano, h.LastDelivered)
|
||||
return err == nil && last.IsZero()
|
||||
}
|
||||
|
||||
// heldConsumers reads the consumers the bus holds on stream, by name.
|
||||
func heldConsumers(c Caller, stream string) (map[string]heldConsumer, error) {
|
||||
raw, err := c.Call(CallStreams, map[string]any{"stream": stream})
|
||||
@@ -161,6 +176,25 @@ func heldConsumers(c Caller, stream string) (map[string]heldConsumer, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heardDurablesOf is, by stream, both twins of every heard entry of every assigned module's membership, the plain
|
||||
// `<consumer>` and the tool runner's `<consumer>_tools` (design 47 H2, H-A-472 tenth and eleventh addenda): each one held
|
||||
// and composed by no pass is judged by the behind check.
|
||||
func heardDurablesOf(recs buscompose.Records) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, node := range sortedKeys(recs.Assigned) {
|
||||
for _, d := range recs.Assigned[node] {
|
||||
for _, h := range buscompose.HeardIssuedFor(node, d) {
|
||||
for _, name := range []string{h.Consumer, h.Consumer + buscompose.ToolsSuffix} {
|
||||
if !slices.Contains(out[h.Stream], name) {
|
||||
out[h.Stream] = append(out[h.Stream], name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// filtersToAdd are the composed filters of a consumer the bus's one lacks, in the composed order; none when the consumer
|
||||
// filters the whole stream (no filters) or holds all.
|
||||
func filtersToAdd(c buscompose.Consumer, h heldConsumer) []string {
|
||||
@@ -200,12 +234,45 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
|
||||
for _, mc := range wanted {
|
||||
byStream[mc.Consumer.Stream] = append(byStream[mc.Consumer.Stream], mc.Consumer)
|
||||
}
|
||||
// The durables no pass composes, the tool runner's twins it makes itself from the membership's spec and the plain one of a
|
||||
// module whose program left, are still judged here, held on the streams each consuming module's heard entries name
|
||||
// (design 47 H2, H-A-472 tenth and eleventh addenda).
|
||||
twins := heardDurablesOf(recs)
|
||||
for stream := range twins {
|
||||
if _, composed := byStream[stream]; !composed {
|
||||
byStream[stream] = nil
|
||||
}
|
||||
}
|
||||
// judge is the behind check of one held consumer: an orphan is left out of it and named in a finding of its own
|
||||
// (H-A-472 (2)), so the alarm says a real backlog only.
|
||||
judge := func(key string, h heldConsumer, held map[string]heldConsumer) {
|
||||
if h.Pending <= BehindBound {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case !p.behindSeen[key]:
|
||||
case orphaned(h, held, p.now(), p.twinCurrentWithin()):
|
||||
orphans = append(orphans, key)
|
||||
default:
|
||||
st.Behind = append(st.Behind, fmt.Sprintf("%s: %d pending", key, h.Pending))
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
for _, stream := range sortedKeys(byStream) {
|
||||
held, err := heldConsumers(p.Call, stream)
|
||||
if err != nil {
|
||||
st.Unread = append(st.Unread, stream+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
composedHere := map[string]bool{}
|
||||
for _, c := range byStream[stream] {
|
||||
composedHere[c.Name] = true
|
||||
}
|
||||
for _, name := range twins[stream] {
|
||||
if h, ok := held[name]; ok && !composedHere[name] {
|
||||
judge(stream+"."+name, h, held)
|
||||
}
|
||||
}
|
||||
for _, c := range byStream[stream] {
|
||||
key := stream + "." + c.Name
|
||||
h, ok := held[c.Name]
|
||||
@@ -213,18 +280,7 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
|
||||
p.sayWorker(key + " is held already")
|
||||
}
|
||||
if ok {
|
||||
if h.Pending > BehindBound {
|
||||
// An orphan is left out of the behind check and named in a finding of its own (H-A-472 (2)), so the
|
||||
// alarm says a real backlog only.
|
||||
switch {
|
||||
case !p.behindSeen[key]:
|
||||
case orphaned(h, held, p.now(), p.twinCurrentWithin()):
|
||||
orphans = append(orphans, key)
|
||||
default:
|
||||
st.Behind = append(st.Behind, fmt.Sprintf("%s: %d pending", key, h.Pending))
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
judge(key, h, held)
|
||||
// A consumer whose module now consumes more events of the stream gains those filters by itself, only ever added:
|
||||
// a filter it holds that is no longer composed stays, and nothing else about it changes.
|
||||
if add := filtersToAdd(c, h); len(add) > 0 && st.On && p.Consumers != nil {
|
||||
@@ -453,8 +509,8 @@ func (p *Part) sayOrphaned(orphans []string) {
|
||||
needsYou(&c, "their removal, with the bus module's nats_consumer_remove", "These durables are more than the bound behind "+
|
||||
"their stream's head with no pulls waiting, while their module's twin on the same stream (<node>_<module> and "+
|
||||
"<node>_<module>_tools) has pulls waiting and delivered within "+p.twinCurrentWithin().String()+" (the setting "+
|
||||
"twin-current-hours): a module that moved into or out of the tool runner left them, and they are not said as "+
|
||||
"messages piling up: "+why)
|
||||
"twin-current-hours): a module that moved into or out of the tool runner left them, or a twin the runner made from "+
|
||||
"the membership and no longer reads, and they are not said as messages piling up: "+why)
|
||||
c.Resolved = "no orphaned durable is left"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user