Compare commits
50
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dc33ae26bb | ||
|
|
a096397928 | ||
|
|
5548fff81f | ||
|
|
52fa062e92 | ||
|
|
ef32d60a5e | ||
|
|
cfd2afe39c | ||
|
|
21daeb8893 | ||
|
|
ae1bf2098d | ||
|
|
f09bd2668a | ||
|
|
82156f8128 | ||
|
|
ffa58e51c4 | ||
|
|
cc1034630f | ||
|
|
73cb9ba690 | ||
|
|
fcac1fc5d0 | ||
|
|
db899d8ad4 | ||
|
|
4ca6e478a6 | ||
|
|
564fd5e0bb | ||
|
|
dc6dfef6f0 | ||
|
|
8eee17a368 | ||
|
|
2b79ee1593 | ||
|
|
9931d4f900 | ||
|
|
18136ce063 | ||
|
|
a5495bb15e | ||
|
|
5d3164b5b7 | ||
|
|
96d8136e2b | ||
|
|
c45f1621fb | ||
|
|
fd2cb13961 | ||
|
|
915a54f136 | ||
|
|
1c0309df23 | ||
|
|
7dd2891f32 | ||
|
|
b7e364e47b | ||
|
|
dca2545eb7 | ||
|
|
bd5852b817 | ||
|
|
1bfd54425b | ||
|
|
c6c86801ca | ||
|
|
33feb148ce | ||
|
|
f44442097c | ||
|
|
1f765ef17b | ||
|
|
203454b1a9 | ||
|
|
b691428ffc | ||
|
|
a41a7c2862 | ||
|
|
7e103f3221 | ||
|
|
c37a7317b7 | ||
|
|
b2c9e60ac3 | ||
|
|
b77b70a66c | ||
|
|
7f5f175b05 | ||
|
|
1f33a4c04f | ||
|
|
49c656a4ce | ||
|
|
9005a2ee8e | ||
|
|
b914974b1d |
@@ -154,6 +154,13 @@ func (b *builtEnds) madeOf(module, fp string) map[string]string {
|
||||
return parts
|
||||
}
|
||||
|
||||
// releasedOf is the version released of module's done build at fp, the release whose parts are what its build.ended made (design
|
||||
// 47 G A-421); false when no such release is heard.
|
||||
func (w *world) releasedOf(module, fp string) (string, bool) {
|
||||
k, ok := w.released.withParts(module, w.ends.madeOf(module, fp))
|
||||
return k.Version, ok && k.Version != ""
|
||||
}
|
||||
|
||||
// of is the version of module released from the source revision, with its parts' digests; false before it is heard.
|
||||
func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bool) {
|
||||
r.mu.Lock()
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
@@ -170,15 +171,21 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
|
||||
}
|
||||
|
||||
// blockedObjects is an object bucket whose listing waits while a gate stands: a keeping that does not end until it is opened.
|
||||
// A listing that meets the gate closes entered: the keeping took its need and runs it (H-A-476).
|
||||
type blockedObjects struct {
|
||||
memObjects
|
||||
mu sync.Mutex
|
||||
gate chan struct{}
|
||||
mu sync.Mutex
|
||||
gate chan struct{}
|
||||
entered chan struct{}
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Names(ctx context.Context) ([]string, error) {
|
||||
b.mu.Lock()
|
||||
gate := b.gate
|
||||
gate, entered := b.gate, b.entered
|
||||
if gate != nil && entered != nil {
|
||||
close(entered)
|
||||
b.entered = nil
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if gate != nil {
|
||||
<-gate
|
||||
@@ -204,16 +211,33 @@ func (b *blockedObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
func (b *blockedObjects) block() func() {
|
||||
gate := make(chan struct{})
|
||||
b.mu.Lock()
|
||||
b.gate = gate
|
||||
b.gate, b.entered = gate, make(chan struct{})
|
||||
b.mu.Unlock()
|
||||
return func() {
|
||||
b.mu.Lock()
|
||||
b.gate = nil
|
||||
b.gate, b.entered = nil, nil
|
||||
b.mu.Unlock()
|
||||
close(gate)
|
||||
}
|
||||
}
|
||||
|
||||
// awaitBlocked returns once a keeping meets the standing gate, so what the keeping took is read after it took it, and fails the
|
||||
// test when none did within bound (H-A-476: a test waits on the signal of the work it reads after, never on an order it assumes).
|
||||
func (b *blockedObjects) awaitBlocked(t *testing.T, bound time.Duration, what string) {
|
||||
t.Helper()
|
||||
b.mu.Lock()
|
||||
entered := b.entered
|
||||
b.mu.Unlock()
|
||||
if entered == nil {
|
||||
return // met already
|
||||
}
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(bound):
|
||||
t.Fatalf("no keeping met the gate within %s: %s", bound, what)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-259, H-A-261: the keeping of the assignment graph runs off the pass loop. At the start, while the first keeping is
|
||||
// blocked, a pass returns and writes no want; the keeping's end wakes a pass, which writes the passed pipeline's want. Later, while a
|
||||
// keeping of new inputs is blocked, a pass writes the wants from the revision it holds, and a pass of the same inputs hands nothing
|
||||
@@ -222,6 +246,7 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
fp := mcc.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
bound := testbus.SetupBound(t) // every wait on a goroutine's work, never a value in the test (H-A-476)
|
||||
var mu sync.Mutex
|
||||
var journal strings.Builder
|
||||
p.logf = func(f string, a ...any) { mu.Lock(); fmt.Fprintf(&journal, f+"\n", a...); mu.Unlock() }
|
||||
@@ -252,7 +277,7 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
go func() { p.pass(context.Background()); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
case <-time.After(bound):
|
||||
t.Fatal("the pass waits on the keeping of the assignment graph")
|
||||
}
|
||||
}
|
||||
@@ -281,7 +306,7 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
open()
|
||||
select {
|
||||
case <-woke:
|
||||
case <-time.After(10 * time.Second):
|
||||
case <-time.After(bound):
|
||||
t.Fatal("the first keeping's end wakes no pass")
|
||||
}
|
||||
pass()
|
||||
@@ -292,7 +317,8 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
open = objects.block() // a keeping of new inputs is blocked
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
pass()
|
||||
pass() // the same inputs again
|
||||
pass() // the same inputs again
|
||||
objects.awaitBlocked(t, bound, "the keeping of new inputs never took its need") // H-A-476: read the need once it was taken
|
||||
if w, err := wantOf("g14"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
|
||||
t.Errorf("the want of web on g14 = %+v (%v), want the passed version while a keeping is blocked", w, err)
|
||||
}
|
||||
|
||||
@@ -656,6 +656,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
// once the first registrations are written (§95, §353).
|
||||
p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf)
|
||||
p.ends.trunk = w.trunkName
|
||||
p.ends.releasedOf = w.releasedOf // a version released before its pipeline passed is taken at the pass (H-A-490)
|
||||
p.ends.runEnded = func(group string) error { return p.dropCandidate(ctx, group) }
|
||||
if err := hearEnds(hub, p.ends); err != nil {
|
||||
logger.Printf("a pipeline's end is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err)
|
||||
|
||||
@@ -409,7 +409,12 @@ func putOfTheBus(t *testing.T) {
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
putModuleRaw(p.w, trunkBranch, "nats", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
|
||||
endDone(t, p.w, "nats", fp, mdigest)
|
||||
// The build and its release (H-A-490 addendum): the put's want names the version released at its parts.
|
||||
built, _ := json.Marshal(builds.BuildEnded{Module: "nats", Fingerprint: fp, Outcome: builds.BuildDone, Manifest: mdigest, Made: map[string]string{"program": "sha256:5"}})
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "nats", Source: 8, Version: "2.10.1", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "nats:program", Digest: "sha256:5"}}})
|
||||
if p.w.ends.put(built) != nil || p.w.released.put(rel) != nil {
|
||||
t.Fatal("the build or its release is not heard")
|
||||
}
|
||||
hkey, _ := nodeengineclient.HaveKey("novox", "nats")
|
||||
before, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
world(buckets.Have, hkey, before)
|
||||
@@ -489,7 +494,7 @@ func putOfTheBus(t *testing.T) {
|
||||
<-woke
|
||||
got = step()
|
||||
w, err := mwc.DecodeWant(wantsKV.values["want.novox.nats"])
|
||||
if got.Step != puts.Wanted || err != nil || w.Fp != fp || !w.Put || !w.Hold {
|
||||
if got.Step != puts.Wanted || err != nil || w.Fp != fp || w.Version != "2.10.1" || !w.Put || !w.Hold {
|
||||
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||
}
|
||||
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
@@ -152,6 +153,10 @@ type pipelineEnds struct {
|
||||
trunk func() string
|
||||
// now is the clock of the tier progress (tiers.go); nil is time.Now.
|
||||
now func() time.Time
|
||||
// releasedOf is the version released of module's build at fp, matched by the parts its build.ended made (world.releasedOf):
|
||||
// a mover whose version was released before its pipeline passed gets its want at the pass (design 47 H2 H-A-490). nil
|
||||
// matches none.
|
||||
releasedOf func(module, fp string) (string, bool)
|
||||
}
|
||||
|
||||
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
|
||||
@@ -161,11 +166,18 @@ func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipel
|
||||
// take keeps, for each mover of a passed pipeline that released a version, that version as the newest of its module on the
|
||||
// pipeline's branch, by compare-and-set on a newer version (design 47 H §750): an older version, or the same one again, writes
|
||||
// nothing. A passed mover with no version yet is said in the journal and taken when the runner says the end again naming it
|
||||
// (H-A-270); it holds its tier in the pipeline's tier progress meanwhile (H-A-277 (1)). It says whether any key changed.
|
||||
// (H-A-270); it holds its tier in the pipeline's tier progress meanwhile (H-A-277 (1)). A mover whose build's version was
|
||||
// released before the pass is taken at the pass with that version (H-A-490). It says whether any key changed.
|
||||
func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" {
|
||||
return false, nil
|
||||
}
|
||||
// The pipeline's key is read by the one parse of the runner's seat's key forms (design 47 H2 H-A-485): a key it does not
|
||||
// know is refused loudly, and nothing of the pipeline is kept, never an empty tier progress.
|
||||
if _, err := pipelinekey.Parse(p.Pipeline); err != nil {
|
||||
e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: %v", p.Pipeline, err)
|
||||
return false, nil
|
||||
}
|
||||
changed := false
|
||||
var kept []tierMover
|
||||
versioned := map[string]bool{}
|
||||
@@ -178,6 +190,14 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
if m.Fingerprint == "" || !moverPassed(m) {
|
||||
continue
|
||||
}
|
||||
if m.Version == "" && e.releasedOf != nil {
|
||||
// A version released before its pipeline passed (an again pipeline on a built, published key) is never written
|
||||
// on the mover, nor its end said again: it is judged at the pass, matched by its build's parts (H-A-490).
|
||||
if v, ok := e.releasedOf(m.Module, m.Fingerprint); ok {
|
||||
e.logf("%s: %s passed; its version %s was released before the pass, taken now", p.Pipeline, m.Module, v)
|
||||
m.Version = v
|
||||
}
|
||||
}
|
||||
if m.Version == "" {
|
||||
// The runner writes a mover's version at its release's version-released, and says this end again once it does
|
||||
// (H-A-270 (1), (2)): the skip is said, never silent.
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -96,6 +97,17 @@ func TestThePassedVersionIsKept(t *testing.T) {
|
||||
ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}},
|
||||
{"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)),
|
||||
ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}},
|
||||
// H-A-485: an again pipeline's end is kept as a take-in's; a key of no form the runner's seat defines is refused, nothing kept.
|
||||
{"an again pipeline keeps its version", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.again.2"
|
||||
return p
|
||||
}()}, &End{Pipeline: "main.5.again.2", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
{"a key of no form is refused", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.later.2"
|
||||
return p
|
||||
}()}, nil},
|
||||
{"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
|
||||
p.Run = "r1"
|
||||
@@ -109,6 +121,9 @@ func TestThePassedVersionIsKept(t *testing.T) {
|
||||
}
|
||||
}
|
||||
got, kept := e.endOf("web", "main")
|
||||
if ks, _ := e.st.keys(); c.name == "a key of no form is refused" && len(ks) != 0 {
|
||||
t.Errorf("%s: the state written holds %v, want nothing (no empty tier progress)", c.name, ks)
|
||||
}
|
||||
switch {
|
||||
case c.want == nil && kept:
|
||||
t.Errorf("%s: kept %+v", c.name, got)
|
||||
@@ -142,10 +157,15 @@ func TestThePassedVersionIsKept(t *testing.T) {
|
||||
// A6/AC1, A5/AC1 (design 47 H §750): a pass writes, on every node the module is assigned to, the want naming the newest version
|
||||
// released on its branch whose pipeline passed, with the pipeline and the rank; with none passed it writes none. No grant,
|
||||
// canary, provider or composition is asked for. H-A-270 (1): a passed end whose mover names no version yet writes no want and
|
||||
// says so in the journal; the same end said again naming the version writes it.
|
||||
// says so in the journal; the same end said again naming the version writes it. H-A-490: a version released before its pipeline
|
||||
// passed (an again pipeline on a built, published key) gets its want at the pass, matched to the mover's build by its parts.
|
||||
func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
for _, passed := range []bool{false, true} {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
passed, releasedBefore bool
|
||||
}{{"no pipeline passed", false, false}, {"passed, then its version released", true, false}, {"released, then its pipeline passed", true, true}} {
|
||||
passed := c.passed
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
@@ -158,7 +178,16 @@ func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
var journal []string
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
|
||||
p.ends.trunk = p.w.trunkName
|
||||
if passed {
|
||||
p.ends.releasedOf = p.w.releasedOf
|
||||
if c.releasedBefore {
|
||||
// Built and released in an earlier pipeline that failed: the runner's again pipeline ends naming no version.
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "web", Source: 4, Version: "0.1.2", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "web:image", Digest: "sha256:7"}}})
|
||||
built, _ := json.Marshal(builds.BuildEnded{Module: "web", Fingerprint: fp, Outcome: builds.BuildDone, Made: map[string]string{"image": "sha256:7"}})
|
||||
if err := p.w.released.put(rel); err != nil || p.w.ends.put(built) != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
|
||||
} else if passed {
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
|
||||
p.wants(context.Background())
|
||||
if len(wantsKV.values) != 0 || !slices.Contains(journal, trunkBranch+".5: web passed with no version yet: taken again when its version is written") {
|
||||
@@ -171,14 +200,14 @@ func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
raw, got := wantsKV.values[key]
|
||||
if got != passed {
|
||||
t.Errorf("passed %v: a want on %s written = %v", passed, node, got)
|
||||
t.Errorf("%s: a want on %s written = %v", c.name, node, got)
|
||||
continue
|
||||
}
|
||||
if !passed {
|
||||
continue
|
||||
}
|
||||
if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 {
|
||||
t.Errorf("the want on %s = %+v (%v)", node, w, err)
|
||||
t.Errorf("%s: the want on %s = %+v (%v)", c.name, node, w, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,6 +181,7 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
|
||||
if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state
|
||||
w.Build, w.BuildWords = e.Outcome, e.Error
|
||||
}
|
||||
w.Version, _ = p.w.releasedOf(module, put.FP) // matched by its build's parts, as the pass matches a mover's (H-A-490 addendum)
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
|
||||
@@ -6,12 +6,14 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
@@ -58,10 +60,16 @@ type tierProgress struct {
|
||||
// tierKey is the key of pipeline's tier progress; pipeline is the runner's key `<branch>.<source>`, its branch escaped.
|
||||
func tierKey(pipeline string) string { return tierPrefix + pipeline }
|
||||
|
||||
// isTierKey says whether a key of the state written is a pipeline's tier progress: three parts, the first `pipeline` (a module's
|
||||
// end has two).
|
||||
// isTierKey says whether a key of the state written is a pipeline's tier progress: `pipeline.` and a pipeline's key of any form
|
||||
// the runner's seat defines, read by the one parse (design 47 H2 H-A-485), so an again pipeline's progress is one as a take-in's
|
||||
// is (a module's end, `<module>.<branch>`, is none).
|
||||
func isTierKey(key string) bool {
|
||||
return strings.HasPrefix(key, tierPrefix) && strings.Count(key, ".") == 2
|
||||
rest, ok := strings.CutPrefix(key, tierPrefix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, err := pipelinekey.Parse(rest)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// startTiers writes the tier progress of a passed pipeline, holding every passed mover, versioned or not, its first tier the
|
||||
@@ -166,11 +174,11 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st
|
||||
continue
|
||||
}
|
||||
pipeline := strings.TrimPrefix(key, tierPrefix)
|
||||
branchEsc, number, _ := strings.Cut(pipeline, ".")
|
||||
branch, err := keys.UnescapeName(branchEsc)
|
||||
pk, err := pipelinekey.Parse(pipeline)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
branch, number := pk.Branch, strconv.FormatInt(pk.Revision, 10)
|
||||
for {
|
||||
t, rev, ok := p.ends.tiersOf(pipeline)
|
||||
if !ok {
|
||||
|
||||
@@ -28,7 +28,12 @@ import (
|
||||
// tier-bound: the finding tier-late names it and its machine, pipeline-wanted says tier-late, and no consumer want is written.
|
||||
func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
db, web := meshcatalogclient.SourcePrefix2+strings.Repeat("d", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64)
|
||||
for _, runs := range []bool{true, false} {
|
||||
// H-A-485: a pipeline started again at the head, keyed `<branch>.<revision>.again.<n>`, moves tier by tier as a take-in's does.
|
||||
for _, c := range []struct {
|
||||
runs bool
|
||||
pipeline string
|
||||
}{{true, trunkBranch + ".5"}, {false, trunkBranch + ".5"}, {true, trunkBranch + ".5.again.2"}, {false, trunkBranch + ".5.again.2"}} {
|
||||
runs, pipeline := c.runs, c.pipeline
|
||||
start := time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC)
|
||||
now := start
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
@@ -41,7 +46,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
var said []meshwants.PipelineWanted
|
||||
p.publishWants = func(event string, body any, _ string) error {
|
||||
if b, ok := body.(meshwants.PipelineWantedEvent); ok {
|
||||
if event != "pipeline-wanted."+trunkBranch+".5" {
|
||||
if event != "pipeline-wanted."+pipeline {
|
||||
t.Errorf("pipeline-wanted said as %s", event)
|
||||
}
|
||||
said = append(said, b)
|
||||
@@ -65,7 +70,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
// The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216).
|
||||
e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
|
||||
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))
|
||||
e.Source = 4
|
||||
e.Source, e.Pipeline = 4, pipeline
|
||||
if _, err := p.ends.take(e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -76,7 +81,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
}
|
||||
p.wants(context.Background())
|
||||
if !wanted("g14", "db") || !wanted("ace", "db") || wanted("g14", "web") || wanted("ace", "web") {
|
||||
t.Fatalf("runs %v: tier 1 alone is not what is wanted first: %v", runs, wantsKV.values)
|
||||
t.Fatalf("%s runs %v: tier 1 alone is not what is wanted first: %v", pipeline, runs, wantsKV.values)
|
||||
}
|
||||
if runs {
|
||||
raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: "db", Fingerprint: db})
|
||||
@@ -86,15 +91,15 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
now = start.Add(time.Minute)
|
||||
p.wants(context.Background())
|
||||
if !wanted("g14", "web") || !wanted("ace", "web") {
|
||||
t.Fatalf("the consumer's want is not written once its provider runs: %v", wantsKV.values)
|
||||
t.Fatalf("%s: the consumer's want is not written once its provider runs: %v", pipeline, wantsKV.values)
|
||||
}
|
||||
raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web})
|
||||
_ = p.w.hearRunning("running.g14.web", raw)
|
||||
p.wants(context.Background())
|
||||
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 {
|
||||
t.Errorf("every tier runs: said %+v, findings %v", said, findings)
|
||||
t.Errorf("%s: every tier runs: said %+v, findings %v", pipeline, said, findings)
|
||||
}
|
||||
if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok {
|
||||
if _, _, ok := p.ends.tiersOf(pipeline); ok {
|
||||
t.Error("the tier progress stands after every tier ran")
|
||||
}
|
||||
continue
|
||||
@@ -107,7 +112,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
now = start.Add(30 * time.Minute)
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
f, ok := findings["delivery."+assign.KeyID(trunkBranch+".5")+".tier-late"]
|
||||
f, ok := findings["delivery."+assign.KeyID(pipeline)+".tier-late"]
|
||||
if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") {
|
||||
t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings)
|
||||
}
|
||||
@@ -116,7 +121,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said)
|
||||
}
|
||||
// A tier progress kept before it held its source says the key's number as both (H-A-216).
|
||||
p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
|
||||
p.sayPipelineWanted(pipeline, trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
|
||||
if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 {
|
||||
t.Errorf("a progress with no source: said %+v, want revision and source 5", said)
|
||||
}
|
||||
|
||||
@@ -34,6 +34,9 @@ type PutWorld struct {
|
||||
Build, BuildWords string
|
||||
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
|
||||
Newest string
|
||||
// Version is the version released at the put's fingerprint, matched by its build's parts as the pass matches a mover's;
|
||||
// "" when none is released (design 47 H2 H-A-490 addendum).
|
||||
Version string
|
||||
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
|
||||
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
|
||||
Composed *ComposedDigest
|
||||
@@ -87,7 +90,10 @@ func StepPut(w PutWorld) PutNext {
|
||||
if h := w.Have; h == nil || h.Fp != p.FP || h.Composed == "" {
|
||||
return say(puts.Refused, "%s runs %s at %s, and its have names no composition at that fingerprint, so the want cannot name what runs", w.Node, w.Module, fp)
|
||||
}
|
||||
p.Composed = w.Have.Composed
|
||||
if w.Version == "" {
|
||||
return say(puts.Refused, "no version released at %s: build or release it first", fp)
|
||||
}
|
||||
p.Composed, p.Version = w.Have.Composed, w.Version
|
||||
words := fmt.Sprintf("the want of %s on %s names %s, the build it runs now (composition %s), from its running statement; the builder's store is not asked",
|
||||
w.Module, w.Node, fp, short(p.Composed))
|
||||
if p.Hold {
|
||||
@@ -115,6 +121,14 @@ func StepPut(w PutWorld) PutNext {
|
||||
default:
|
||||
return say(puts.Build, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
|
||||
}
|
||||
if p.Step != puts.Wanted && p.Step != puts.Holding {
|
||||
// The want names the build's released version beside its fingerprint, which the node-engine composes from (H-A-490
|
||||
// addendum): a build with none released is refused, never wanted by its fingerprint alone.
|
||||
if w.Version == "" {
|
||||
return say(puts.Refused, "no version released at %s: build or release it first", fp)
|
||||
}
|
||||
p.Version = w.Version
|
||||
}
|
||||
if w.Composed == nil || !w.Granted {
|
||||
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
|
||||
if w.Composed != nil {
|
||||
@@ -212,9 +226,10 @@ func needsPerson(w PutWorld, d ComposedDigest) bool {
|
||||
return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot
|
||||
}
|
||||
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint, marked put, and held when it holds.
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint and the version released at it, marked put, and held when
|
||||
// it holds.
|
||||
func PutWant(p puts.Put) mwc.Want {
|
||||
return mwc.Want{Fp: p.FP, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
return mwc.Want{Fp: p.FP, Version: p.Version, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
}
|
||||
|
||||
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
|
||||
|
||||
@@ -105,7 +105,7 @@ func Wants(m Module, wants map[string]mwc.Want) Outcome {
|
||||
if !put && had && before.Put && before.Hold {
|
||||
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard; only a put
|
||||
// heard released gives it back to the rule.
|
||||
p, put = puts.Put{FP: before.Fp, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), Step: puts.Holding}, true
|
||||
p, put = puts.Put{FP: before.Fp, Version: before.Version, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), Step: puts.Holding}, true
|
||||
}
|
||||
if put {
|
||||
if want, owns := PutOwns(p); owns {
|
||||
|
||||
@@ -92,7 +92,7 @@ func TestAWantNamesTheNewestPassedVersion(t *testing.T) {
|
||||
// want stands, marked put and held, and once released the rule writes the passed version again.
|
||||
func TestAPutOwnsItsNodesWant(t *testing.T) {
|
||||
m := released()
|
||||
put := puts.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||
put := puts.Put{FP: fNew, Version: "1.2.3", Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
step string
|
||||
@@ -100,7 +100,7 @@ func TestAPutOwnsItsNodesWant(t *testing.T) {
|
||||
want *mwc.Want // nil: nothing written on ace
|
||||
}{
|
||||
{"waiting for the operator", puts.Person, wantedOld(), nil},
|
||||
{"wanted", puts.Wanted, wantedOld(), &mwc.Want{Fp: fNew, Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||
{"wanted", puts.Wanted, wantedOld(), &mwc.Want{Fp: fNew, Version: "1.2.3", Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||
{"holding, written already", puts.Holding, map[string]mwc.Want{"ace": PutWant(put)}, nil},
|
||||
{"released: the rule again", puts.Released, map[string]mwc.Want{"ace": PutWant(put)}, &mwc.Want{Fp: fNew, Version: "1.2.3", Pipeline: "main.12", Rank: 2}},
|
||||
{"not heard: a held want at an older fingerprint stands", "", map[string]mwc.Want{"ace": {Fp: fOld, Put: true, Hold: true, Words: "put: back"}}, nil},
|
||||
@@ -144,7 +144,7 @@ func stepsOfAPut(t *testing.T) {
|
||||
running := nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
|
||||
bus := func() PutWorld {
|
||||
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: builds.BuildDone, Newest: fNew,
|
||||
Composed: &d, Granted: true, Have: &running, Now: now}
|
||||
Version: "1.2.3", Composed: &d, Granted: true, Have: &running, Now: now}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -172,6 +172,16 @@ func stepsOfAPut(t *testing.T) {
|
||||
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
|
||||
return w
|
||||
}, puts.Refused, "names no composition", false, false, false},
|
||||
// H-A-490 (addendum): a want names the build's released version, so a done build with none released is refused, and so
|
||||
// is the build the node runs now when no release names its parts.
|
||||
{"a done build with no released version: refused", func() PutWorld { w := bus(); w.Version = ""; return w }, puts.Refused, "no version released at", false, false, false},
|
||||
{"the build the node runs now, no version released: refused", func() PutWorld {
|
||||
w := bus()
|
||||
w.Build, w.Newest, w.Version = "", fpOf('c'), ""
|
||||
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
|
||||
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
|
||||
return w
|
||||
}, puts.Refused, "build or release it first", false, false, false},
|
||||
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = builds.BuildFailed, "go vet"; return w }, puts.Refused, "go vet", false, false, false},
|
||||
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, puts.Compose, "no composition of nats for novox", false, false, false},
|
||||
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, puts.Compose, "does not name novox", false, false, false},
|
||||
@@ -256,6 +266,9 @@ func stepsOfAPut(t *testing.T) {
|
||||
}
|
||||
for _, c := range cases {
|
||||
next := StepPut(c.world())
|
||||
if (next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) && c.world().Put.Step == puts.Asked && next.Put.Version != "1.2.3" {
|
||||
t.Errorf("put, %s: its want names version %q, want the build's released 1.2.3", c.name, next.Put.Version)
|
||||
}
|
||||
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
|
||||
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
|
||||
}
|
||||
|
||||
@@ -65,6 +65,9 @@ type Put struct {
|
||||
V int `json:"v"`
|
||||
// FP is the fingerprint put, of the new form.
|
||||
FP string `json:"fp"`
|
||||
// Version is the version released at FP, which the want names beside it, once the put found it (design 47 H2 H-A-490
|
||||
// addendum).
|
||||
Version string `json:"version,omitempty"`
|
||||
// Hold keeps the want at FP until released.
|
||||
Hold bool `json:"hold,omitempty"`
|
||||
Why string `json:"why"`
|
||||
|
||||
@@ -5,9 +5,12 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -54,3 +57,14 @@ func TestABuildSeatsStepsAreReadInItsHoldersState(t *testing.T) {
|
||||
t.Errorf("a seat with no holder reads %v; want an error naming it", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-487: a build key's time is its entry's own write time, read with the key and no history.
|
||||
func TestABuildKeysTimeIsItsEntrysOwnWriteTime(t *testing.T) {
|
||||
at := time.Date(2026, 11, 22, 20, 45, 0, 0, time.UTC)
|
||||
raw, rev, got, err := entryOf(func(key string) (*stdio.StateEntry, error) {
|
||||
return &stdio.StateEntry{Key: key, Value: json.RawMessage(`{}`), Revision: 4, At: at}, nil
|
||||
})("k")
|
||||
if err != nil || string(raw) != "{}" || rev != 4 || !got.Equal(at) {
|
||||
t.Errorf("entry = %s rev %d at %v (%v); want the entry's own write time %v", raw, rev, got, err, at)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,7 +430,7 @@ func main() {
|
||||
return a, json.Unmarshal(raw, &a)
|
||||
}
|
||||
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr, reserve)
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds").Get), Asked: read(stdio.State("asked")),
|
||||
Newest: newestVia(stdio.Ask), Steps: r}))
|
||||
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
as := stdio.State(versions.ActedState)
|
||||
@@ -481,13 +481,14 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// entryOf reads a key with its revision and when it was written.
|
||||
func entryOf(s stdio.KeptState) func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
// entryOf reads a key with its revision and when it was written, the entry's own write time: no key's history is read
|
||||
// (design 47 H2 H-A-487).
|
||||
func entryOf(get func(key string) (*stdio.StateEntry, error)) func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
return func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
e, err := s.Get(key)
|
||||
e, err := get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, time.Time{}, err
|
||||
}
|
||||
return e.Value, e.Revision, written(s, key, e.Revision), nil
|
||||
return e.Value, e.Revision, e.At, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func publishQueueVia(ask func(key string, body any) (json.RawMessage, error)) fu
|
||||
}
|
||||
}
|
||||
|
||||
// publishActTools are builder_publish_drop and builder_publish_first (design 47 H2 §793): hand acts, recorded in the hand-act log
|
||||
// publishActTools are builder_publish_drop (design 47 H2 §793; builder_publish_first is retired, H-A-483 third addendum): hand acts, recorded in the hand-act log
|
||||
// first, then written to reasks for the builder, which owns the work queue, to perform and answer.
|
||||
func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
@@ -108,7 +108,6 @@ func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclie
|
||||
}
|
||||
return []stdio.Tool{
|
||||
tool(versions.PublishDropTool, build.ModePublishDrop, versions.PublishDropDescription, "dropped the publish of"),
|
||||
tool(versions.PublishFirstTool, build.ModePublishFirst, versions.PublishFirstDescription, "put first the publish of"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,8 +78,8 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
}
|
||||
})
|
||||
// Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from
|
||||
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and
|
||||
// builder_publish_first record the hand act first, then write their reasks value.
|
||||
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop records the hand act
|
||||
// first, then writes its reasks value; builder_publish_first is retired (H-A-483, third addendum).
|
||||
t.Run("the queue is the builder's verb publish-queue", func(t *testing.T) {
|
||||
var asked []string
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -123,7 +123,7 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
t.Errorf("asked %v, answered %+v (%v): want each holder asked in turn at its node, answered together, the silent one with why", asked, hs, err)
|
||||
}
|
||||
})
|
||||
t.Run("drop and first record the hand act, then ask the builder", func(t *testing.T) {
|
||||
t.Run("drop records the hand act, then asks the builder; first is retired", func(t *testing.T) {
|
||||
var order []string
|
||||
ask := func(key string, value []byte) (json.RawMessage, error) {
|
||||
order = append(order, "ask "+key)
|
||||
@@ -141,8 +141,8 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
t.Fatalf("%s: %v", tool.Name, err)
|
||||
}
|
||||
}
|
||||
if len(tools) != 2 || fmt.Sprint(order) != "[record ask publish-drop.5000000000 record ask publish-first.5000000000]" {
|
||||
t.Errorf("tools %d, order %v: want each hand act recorded before its reasks value", len(tools), order)
|
||||
if len(tools) != 1 || tools[0].Name != versions.PublishDropTool || fmt.Sprint(order) != "[record ask publish-drop.5000000000]" {
|
||||
t.Errorf("tools %d, order %v: want builder_publish_drop alone, its hand act recorded before its reasks value", len(tools), order)
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"module": "m", "why": "no version"}); err == nil {
|
||||
t.Error("a drop naming no version is not refused")
|
||||
|
||||
@@ -40,6 +40,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
@@ -578,7 +579,10 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
first := "src2:" + strings.Repeat("1", 64)
|
||||
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, first, 1)
|
||||
digest := b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, fp, 2)
|
||||
b.start(t, "n1", map[string]string{"MESH_BUILDER_STEP_BOUND": "1s"})
|
||||
// H-A-473 (addendum): the builds that must end done run under the level bound's set-up share, never a bound a loaded
|
||||
// node cannot meet; the hung step's part below starts the builder again at a short bound of its own.
|
||||
doneBound := map[string]string{"MESH_BUILDER_STEP_BOUND": testbus.SetupBound(t).String()}
|
||||
b.start(t, "n1", doneBound)
|
||||
v := b.await(t, "netcheck", isState(builds.BuildDone))
|
||||
for _, a := range []string{"tools-go", "tools-typescript"} {
|
||||
es := reg.of(t, "netcheck/"+a)
|
||||
@@ -693,6 +697,8 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
}
|
||||
// DLV-B10: a build module's bound running out fails the trunk build, which is the builder's finding build-failed; the
|
||||
// fingerprint built again clears it with its done build, numbering nothing of the builder's own (H-A-159).
|
||||
b.stop()
|
||||
b.start(t, "n1", map[string]string{"MESH_BUILDER_STEP_BOUND": "1s"}) // a step that never ends runs out of any bound
|
||||
release := make(chan struct{})
|
||||
b.fake.hang.Store(&release)
|
||||
third := "src2:" + strings.Repeat("4", 64)
|
||||
@@ -713,6 +719,8 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
reg.awaitFinding(t, failed, true)
|
||||
b.fake.hang.Store(nil)
|
||||
close(release)
|
||||
b.stop() // the build again must end done
|
||||
b.start(t, "n1", doneBound)
|
||||
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, "89abcdef0123456789abcdef0123456789abcdef", third, 3) // the entry moves to a later commit, same fingerprint
|
||||
b.awaitFP(t, "netcheck", third, isState(builds.BuildDone))
|
||||
reg.awaitFinding(t, failed, false)
|
||||
|
||||
@@ -233,6 +233,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
b.PackageWait = time.Duration(packageWait) * time.Minute
|
||||
// A version's digests are recorded under the builder's own bound, never the build's (H-A-492); it is record-bound-seconds.
|
||||
recordBound, err := numberFile(getenv("MESH_BUILDER_RECORD_BOUND_FILE"), int(build.DefaultRecordBound/time.Second))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.RecordBound = time.Duration(recordBound) * time.Second
|
||||
// Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194).
|
||||
b.PublishedVersions = publishedVersionsVia(calls.Call)
|
||||
waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
|
||||
@@ -254,11 +260,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// A package's number reserved is said as the seat mesh-builds' version-reserved, so the indexer re-pins at the reservation (§805).
|
||||
reserved.names = versions.NamesFrom(versions.EntriesOf(branches))
|
||||
reserved.r.Names = reserved.names // a Go package's module path names its tag's directory (§796)
|
||||
reserved.sayReserved = sayReservedOn(js, cred)
|
||||
b.Reserved = reserved
|
||||
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
|
||||
reserved.withdraw = b.Withdraw
|
||||
reserved.recordAgain = b.RecordAgain // a done build's digests its end did not record are recorded at the start (H-A-492)
|
||||
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
|
||||
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
@@ -273,7 +279,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk),
|
||||
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
|
||||
Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
|
||||
Started: time.Now()} // a build ended from now on is of the live class, sent first (H-A-290 (1))
|
||||
Started: time.Now()} // a build ended from now on is of the live class (H-A-290 (1))
|
||||
// A number a registry seat already holds is never reserved (H-A-275): read before reserving, through the seats' verb versions.
|
||||
reserved.r.HeldNumbers = func(ctx context.Context, module, fp string, refs []string) ([]string, error) {
|
||||
return republish.HeldNumbers(ctx, module, fp, refs, registryVersionsVia(calls.Call))
|
||||
@@ -322,15 +328,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)).
|
||||
republish.Ended = items.Ended
|
||||
pq.take = items.take
|
||||
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
|
||||
// At the start, a built reservation the catalog holds released is taken in as its version-released would (H-A-484 addendum),
|
||||
// before the publish items of the rest are asked.
|
||||
reasksKV.Ready(func(jetstream.KeyValue) {
|
||||
reserved.releasedAtStart(ctx)
|
||||
items.AskAllDone(ctx)
|
||||
})
|
||||
wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept)
|
||||
acts.Republish = republish.Republish
|
||||
acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) {
|
||||
return pq.drop(ctx, items, module, version, by, why)
|
||||
}
|
||||
acts.PublishFirst = func(ctx context.Context, module, version, _, _ string) (string, error) {
|
||||
return pq.first(ctx, items, module, version)
|
||||
}
|
||||
// builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does.
|
||||
acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf)
|
||||
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
|
||||
|
||||
+413
-266
@@ -22,19 +22,21 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder.
|
||||
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
|
||||
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the
|
||||
// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts
|
||||
// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit, its
|
||||
// reservation's version-reserved waited on by name (§821), and
|
||||
// §794's backlog held six minutes, as steps (H-A-453 (3)).
|
||||
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, one pull consumer made as it learns a holder
|
||||
// (H-A-483 and its addenda), the class durables of an earlier release retired. No item is lost, a failed send comes back after
|
||||
// its pause behind an item put meanwhile, and a dropped item never comes back; 100 again items and a live one put while the
|
||||
// builder was away are all sent, the dropped one never, and put again within the duplicates window it is not stored; the move
|
||||
// of the old state run twice puts each item once, and the builder removes the old state's bucket itself (H-A-461 (1)). The seat
|
||||
// mesh-builds' verb publish-queue reads the stream: counts per holder and class, the oldest and newest, the items listed in
|
||||
// stream order (H-A-454; H-A-483, third addendum). Then §796's tag at another commit, its reservation's version-reserved waited on by name (§821), and §794's
|
||||
// backlog held six minutes, as steps (H-A-453 (3)).
|
||||
func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
b := startBed(t)
|
||||
ctx := b.ctx
|
||||
// The stream as the manifest's old streams section declares it: the module's own, retention work-queue, duplicates 24 hours.
|
||||
@@ -43,281 +45,426 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
var mu sync.Mutex
|
||||
var sent []string
|
||||
published := map[string]bool{} // the modules whose publish is heard
|
||||
sentCh := make(chan string, 200)
|
||||
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5},
|
||||
now: time.Now, logf: t.Logf,
|
||||
reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) {
|
||||
// One consumer per holder (H-A-483 and its addenda): the holder's class durables of an earlier release are retired, the item
|
||||
// one of them held unacknowledged reaches the one consumer; no item is lost; a failed send comes back after its pause, and a
|
||||
// live item put while it waits is sent before it returns; a dropped item never comes back.
|
||||
t.Run("one consumer", func(t *testing.T) {
|
||||
const holder, pause = "paused", 3 * time.Second // the pause is far longer than a send of an item put now
|
||||
bound := testbus.SetupBound(t)
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
type send struct {
|
||||
module string
|
||||
at time.Time
|
||||
}
|
||||
var mu sync.Mutex
|
||||
published, failed := map[string]bool{}, map[string]bool{}
|
||||
sentCh := make(chan send, 16)
|
||||
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: pause, PauseMax: time.Minute, Tries: 5},
|
||||
now: time.Now, logf: t.Logf,
|
||||
reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil
|
||||
},
|
||||
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
|
||||
sentCh <- send{q.Module, time.Now()}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if (q.Module == "fail" || q.Module == "dropme") && !failed[q.Module] { // the first send of each fails
|
||||
failed[q.Module] = true
|
||||
return versions.SendFate{Failed: "the registry is down"}
|
||||
}
|
||||
published[q.Module] = true
|
||||
return versions.SendFate{Sent: true}
|
||||
}}
|
||||
pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")}
|
||||
put := func(module, class string) {
|
||||
q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()}
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, pq.subject(bind, q), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
hear := func() { // each publish sent is heard: its item acknowledged once it is held
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil
|
||||
},
|
||||
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
|
||||
mu.Lock()
|
||||
sent = append(sent, q.Module)
|
||||
var done []string
|
||||
for m, ok := range published {
|
||||
if ok {
|
||||
done = append(done, m)
|
||||
}
|
||||
}
|
||||
mu.Unlock()
|
||||
sentCh <- q.Module
|
||||
return versions.SendFate{Sent: true}
|
||||
}}
|
||||
pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")}
|
||||
item := func(module, class string) versions.QueuedPublish {
|
||||
return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()}
|
||||
}
|
||||
away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil {
|
||||
for _, m := range done {
|
||||
items.heard(ctx, m, "0.1.0")
|
||||
}
|
||||
}
|
||||
next := func() send {
|
||||
t.Helper()
|
||||
select {
|
||||
case s := <-sentCh:
|
||||
hear()
|
||||
return s
|
||||
case <-time.After(bound):
|
||||
t.Fatal("no item was sent within the level's set-up share")
|
||||
}
|
||||
return send{}
|
||||
}
|
||||
// The two class durables an earlier release made, the live one holding an item given and not acknowledged.
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
if _, err := b.js.CreateOrUpdateConsumer(ctx, bind.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder) + "-" + class,
|
||||
FilterSubject: bind.Subject + ".publish." + holder + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy, AckWait: time.Hour}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put("given", versions.PublishClassLive)
|
||||
old, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder)+"-"+versions.PublishClassLive)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for i := range 100 {
|
||||
away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart))
|
||||
}
|
||||
away(item("dropme", versions.PublishClassStart))
|
||||
away(item("live", versions.PublishClassLive))
|
||||
if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil {
|
||||
t.Fatalf("drop: %v", err)
|
||||
} else {
|
||||
t.Log(answer)
|
||||
}
|
||||
if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup {
|
||||
t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err)
|
||||
}
|
||||
raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`))
|
||||
if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart ||
|
||||
a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap ||
|
||||
a.Queue[0].Module != "live" || a.Capped {
|
||||
t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err)
|
||||
}
|
||||
if err := pq.learnAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case first := <-sentCh:
|
||||
if first != "live" {
|
||||
t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first)
|
||||
if m, err := old.Next(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if md, _ := m.Metadata(); md.NumDelivered != 1 {
|
||||
t.Fatalf("the old live durable gave %+v", md)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("nothing was sent")
|
||||
}
|
||||
for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the again items were not sent")
|
||||
pq.learn(ctx, holder)
|
||||
stream, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for n := range stream.ConsumerNames(ctx).Name() {
|
||||
if strings.HasPrefix(n, publishesConsumer(holder)) {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(names, []string{publishesConsumer(holder)}) {
|
||||
t.Fatalf("the consumers of %s are %v, want only %s: the class durables retired", holder, names, publishesConsumer(holder))
|
||||
}
|
||||
if s := next(); s.module != "given" {
|
||||
t.Fatalf("sent %s first, want the item the old durable held unacknowledged", s.module)
|
||||
}
|
||||
put("fail", versions.PublishClassStart)
|
||||
put("dropme", versions.PublishClassStart)
|
||||
var firstFail time.Time
|
||||
for range 2 {
|
||||
if s := next(); s.module == "fail" {
|
||||
firstFail = s.at
|
||||
}
|
||||
}
|
||||
put("live", versions.PublishClassLive) // put while both failed items wait out their pause
|
||||
if s := next(); s.module != "live" {
|
||||
t.Fatalf("sent %s, want the live item put while the failed ones wait their pause", s.module)
|
||||
}
|
||||
if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil {
|
||||
t.Fatalf("drop: %v", err)
|
||||
} else {
|
||||
t.Log(answer)
|
||||
}
|
||||
if s := next(); s.module != "fail" || s.at.Sub(firstFail) < pause {
|
||||
t.Fatalf("sent %s %s after the failed send, want fail again after its pause %s", s.module, s.at.Sub(firstFail), pause)
|
||||
}
|
||||
// Every item is acknowledged and the dropped one is gone: nothing is left to come back.
|
||||
c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(bound); ; {
|
||||
hear()
|
||||
info, err := c.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sinfo, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish."+holder+".>"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumAckPending == 0 && info.NumPending == 0 && len(sinfo.State.Subjects) == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the queue of %s holds %v, its consumer %d pending and %d unacknowledged: want every item acknowledged and the dropped one gone", holder, sinfo.State.Subjects, info.NumPending, info.NumAckPending)
|
||||
}
|
||||
select {
|
||||
case s := <-sentCh:
|
||||
t.Fatalf("sent %s again, want nothing more", s.module)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the queue, the move, tags and backlog", func(t *testing.T) {
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
var mu sync.Mutex
|
||||
var sent []string
|
||||
published := map[string]bool{} // the modules whose publish is heard
|
||||
sentCh := make(chan string, 200)
|
||||
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5},
|
||||
now: time.Now, logf: t.Logf,
|
||||
reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil
|
||||
},
|
||||
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
|
||||
mu.Lock()
|
||||
sent = append(sent, q.Module)
|
||||
mu.Unlock()
|
||||
sentCh <- q.Module
|
||||
return versions.SendFate{Sent: true}
|
||||
}}
|
||||
pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")}
|
||||
item := func(module, class string) versions.QueuedPublish {
|
||||
return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()}
|
||||
}
|
||||
away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for i := range 100 {
|
||||
away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart))
|
||||
}
|
||||
away(item("dropme", versions.PublishClassStart))
|
||||
away(item("live", versions.PublishClassLive))
|
||||
if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil {
|
||||
t.Fatalf("drop: %v", err)
|
||||
} else {
|
||||
t.Log(answer)
|
||||
}
|
||||
if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup {
|
||||
t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err)
|
||||
}
|
||||
raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`))
|
||||
if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart ||
|
||||
a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap ||
|
||||
a.Queue[0].Module != "a000" || a.Capped {
|
||||
t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed in stream order, a000 first", raw, err)
|
||||
}
|
||||
if err := pq.learnAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 4 { // the items in flight: at most four held unacknowledged, their publish unheard
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the again items were not sent")
|
||||
}
|
||||
}
|
||||
deadline := time.After(60 * time.Second)
|
||||
for { // each publish sent is heard: its item acknowledged, the next sent
|
||||
mu.Lock()
|
||||
n, done := len(sent), slices.Clone(sent)
|
||||
for _, m := range done {
|
||||
published[m] = true
|
||||
}
|
||||
mu.Unlock()
|
||||
if n >= 101 {
|
||||
break
|
||||
}
|
||||
for _, m := range done {
|
||||
items.heard(ctx, m, "0.1.0")
|
||||
}
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-deadline:
|
||||
t.Fatalf("sent %d of 101", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
deadline := time.After(60 * time.Second)
|
||||
for { // each publish sent is heard: its item acknowledged, the next sent
|
||||
mu.Lock()
|
||||
n, done := len(sent), slices.Clone(sent)
|
||||
for _, m := range done {
|
||||
published[m] = true
|
||||
if slices.Contains(sent, "dropme") || len(sent) != 101 {
|
||||
t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme"))
|
||||
}
|
||||
mu.Unlock()
|
||||
if n >= 101 {
|
||||
break
|
||||
|
||||
// The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked).
|
||||
old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range done {
|
||||
items.heard(ctx, m, "0.1.0")
|
||||
var tagsWritten []string
|
||||
call := func(tool string, body any) (json.RawMessage, error) {
|
||||
switch tool {
|
||||
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit, never read (H-A-494)
|
||||
tagsWritten = append(tagsWritten, "read "+body.(gitseat.TagsInput).Prefix)
|
||||
return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}})
|
||||
case "seat:" + gitseat.Seat + ".tag":
|
||||
tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit)
|
||||
return json.Marshal(gitseat.TagAnswer{Performed: true})
|
||||
}
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
||||
}
|
||||
found := &memFindings{vals: map[string][]byte{}}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
type oldItem struct {
|
||||
key string
|
||||
raw []byte
|
||||
}
|
||||
var olds []oldItem
|
||||
for _, m := range []string{"m1", "m2", "m3"} {
|
||||
v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"})
|
||||
if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw})
|
||||
}
|
||||
rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) {
|
||||
data, _ := json.Marshal(q)
|
||||
ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return ack.Duplicate, nil
|
||||
}, Started: time.Now(), Logf: t.Logf}
|
||||
oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)
|
||||
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
|
||||
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err)
|
||||
}
|
||||
// A start that stopped before the removal moves the same items again: the duplicates window keeps each once.
|
||||
again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range olds {
|
||||
if _, err := again.Put(ctx, e.key, e.raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
|
||||
moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves
|
||||
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Errorf("after the second move the old state's bucket answers %v, want it removed", err)
|
||||
}
|
||||
st, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 {
|
||||
t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects)
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-494: no git tag is read before a reserve nor written at a release: m4's 0.2.0, which the seat git would
|
||||
// answer tagged at another commit, is reserved, no finding raised, and its version-released writes no tag.
|
||||
// Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it
|
||||
// (sayReservedOn), and waited on by the seat and the event's name.
|
||||
rv.names = func(context.Context, string, string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/m4", true, nil
|
||||
}
|
||||
rv.sayReserved = sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"})
|
||||
said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer said.Close()
|
||||
reservedCh := make(chan *nats.Msg, 4)
|
||||
rs, err := said.ChanSubscribe(meshbuilds.SubjectVersionReserved, reservedCh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rs.Unsubscribe()
|
||||
if err := said.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4",
|
||||
Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1})
|
||||
elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere)
|
||||
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.0" || v4.Tag != "" || raised {
|
||||
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.0 naming no tag, no tag read and no finding", v4, err, elsewhere, raised)
|
||||
}
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-deadline:
|
||||
t.Fatalf("sent %d of 101", n)
|
||||
case m := <-reservedCh:
|
||||
ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data)
|
||||
if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.0" ||
|
||||
ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule {
|
||||
t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.0, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source"))
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved)
|
||||
}
|
||||
catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := func(module, version string) jetstream.Msg {
|
||||
t.Helper()
|
||||
data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version})
|
||||
if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
if !rv.heard(ctx, next("m4", "0.2.0")) {
|
||||
t.Fatal("the reservations did not take version-released")
|
||||
}
|
||||
if _, standing, _ := rv.r.Of(ctx, "m4", "0.2.0"); standing || len(tagsWritten) != 0 {
|
||||
t.Errorf("after version-released m4 0.2.0 standing %v, tags read or written %v: want it released and no tag read or written", standing, tagsWritten)
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
if slices.Contains(sent, "dropme") || len(sent) != 101 {
|
||||
t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme"))
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
// The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked).
|
||||
old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var tagsWritten []string
|
||||
call := func(tool string, body any) (json.RawMessage, error) {
|
||||
switch tool {
|
||||
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796)
|
||||
return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}})
|
||||
case "seat:" + gitseat.Seat + ".tag":
|
||||
tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit)
|
||||
return json.Marshal(gitseat.TagAnswer{Performed: true})
|
||||
}
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
||||
}
|
||||
found := &memFindings{vals: map[string][]byte{}}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
type oldItem struct {
|
||||
key string
|
||||
raw []byte
|
||||
}
|
||||
var olds []oldItem
|
||||
for _, m := range []string{"m1", "m2", "m3"} {
|
||||
v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m})
|
||||
// Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its
|
||||
// consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five
|
||||
// minutes it does not.
|
||||
consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
held := time.Now()
|
||||
hub := &oneHandler{}
|
||||
aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held },
|
||||
sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}}
|
||||
_ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again
|
||||
hub.handle(next("m5", "0.1.0"))
|
||||
state := versions.Reader{Keys: func() ([]string, error) {
|
||||
l, err := consumers.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var keys []string
|
||||
for k := range l.Keys() {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys, nil
|
||||
}, Get: func(k string) (json.RawMessage, bool, error) {
|
||||
e, err := consumers.Get(ctx, k)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value(), true, nil
|
||||
}}
|
||||
if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") {
|
||||
t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a)
|
||||
}
|
||||
raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"})
|
||||
if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy {
|
||||
t.Errorf("the health four minutes in = %+v, want healthy", a)
|
||||
}
|
||||
olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw})
|
||||
}
|
||||
rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) {
|
||||
data, _ := json.Marshal(q)
|
||||
ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return ack.Duplicate, nil
|
||||
}, Started: time.Now(), Logf: t.Logf}
|
||||
oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)
|
||||
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
|
||||
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err)
|
||||
}
|
||||
// A start that stopped before the removal moves the same items again: the duplicates window keeps each once.
|
||||
again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range olds {
|
||||
if _, err := again.Put(ctx, e.key, e.raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
|
||||
moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves
|
||||
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Errorf("after the second move the old state's bucket answers %v, want it removed", err)
|
||||
}
|
||||
st, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 {
|
||||
t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects)
|
||||
}
|
||||
|
||||
// Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another
|
||||
// commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags <directory>/v<version> at
|
||||
// the build's commit, and the module's version-released clears the finding.
|
||||
rv.r.Names = func(context.Context, string, string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/m4", true, nil
|
||||
}
|
||||
// Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it
|
||||
// (sayReservedOn), and waited on by the seat and the event's name.
|
||||
rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"})
|
||||
said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer said.Close()
|
||||
reservedCh := make(chan *nats.Msg, 4)
|
||||
rs, err := said.ChanSubscribe(meshbuilds.SubjectVersionReserved, reservedCh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rs.Unsubscribe()
|
||||
if err := said.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib",
|
||||
Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1})
|
||||
elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere)
|
||||
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised {
|
||||
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised)
|
||||
}
|
||||
select {
|
||||
case m := <-reservedCh:
|
||||
ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data)
|
||||
if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" ||
|
||||
ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule {
|
||||
t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source"))
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved)
|
||||
}
|
||||
catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := func(module, version string) jetstream.Msg {
|
||||
t.Helper()
|
||||
data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version})
|
||||
if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
if !rv.heard(ctx, next("m4", "0.2.1")) {
|
||||
t.Fatal("the reservations did not take version-released")
|
||||
}
|
||||
if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) {
|
||||
t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten)
|
||||
}
|
||||
|
||||
// Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its
|
||||
// consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five
|
||||
// minutes it does not.
|
||||
consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := time.Now()
|
||||
hub := &oneHandler{}
|
||||
aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held },
|
||||
sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}}
|
||||
_ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again
|
||||
hub.handle(next("m5", "0.1.0"))
|
||||
state := versions.Reader{Keys: func() ([]string, error) {
|
||||
l, err := consumers.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var keys []string
|
||||
for k := range l.Keys() {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys, nil
|
||||
}, Get: func(k string) (json.RawMessage, bool, error) {
|
||||
e, err := consumers.Get(ctx, k)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value(), true, nil
|
||||
}}
|
||||
if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") {
|
||||
t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a)
|
||||
}
|
||||
if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy {
|
||||
t.Errorf("the health four minutes in = %+v, want healthy", a)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -531,9 +530,9 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
|
||||
// the manifest's old streams section with the work-queue retention, the module's own, consumed by the module's one process on the
|
||||
// module's account (H-A-464), which declares no bus and so is the stream's one reader by the module's shape; each item on
|
||||
// `mesh.stream.<module>.event.publish.<holder>.<class>.<module>.<version>` under the message id
|
||||
// `<holder>.<class>.<module>.<version>`. The builder makes the pull consumers at run time as it learns a holder,
|
||||
// `publishes-<holder>-<class>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live
|
||||
// class first, at most four sends at once per holder.
|
||||
// `<holder>.<class>.<module>.<version>`. The builder makes one pull consumer at run time as it learns a holder,
|
||||
// `publishes-<holder>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and sends its items in the
|
||||
// order the consumer gives them, at most four at once per holder (H-A-483 and its addenda).
|
||||
type publishQueue struct {
|
||||
js jetstream.JetStream
|
||||
binding func() (publishesStream, error)
|
||||
@@ -555,16 +554,16 @@ func publishesOf(module string) func() (publishesStream, error) {
|
||||
return func() (publishesStream, error) { return b, nil }
|
||||
}
|
||||
|
||||
// The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's
|
||||
// largest; the builder ends an item at publish-tries anyway.
|
||||
// The consumer's numbers (H-A-446 (2)): at most four sends in flight per holder, and ten deliveries, the work queue's largest;
|
||||
// the builder ends an item at publish-tries anyway.
|
||||
const (
|
||||
publishesMaxAckPending = versions.PublishSendsAtOnce
|
||||
publishesMaxDeliver = 10
|
||||
)
|
||||
|
||||
// publishesConsumer is the durable consumer of holder's items of class.
|
||||
func publishesConsumer(holder, class string) string {
|
||||
return versions.PublishesLocal + "-" + holder + "-" + class
|
||||
// publishesConsumer is the one durable consumer of holder's items (H-A-483 addendum).
|
||||
func publishesConsumer(holder string) string {
|
||||
return versions.PublishesLocal + "-" + holder
|
||||
}
|
||||
|
||||
// subject is the item's subject on the bus.
|
||||
@@ -616,8 +615,15 @@ func (pq *publishQueue) learnAll(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// learn starts holder's pull once.
|
||||
// learn starts holder's pull once. A holder is a module name, so one ending in -live or -again, the class durables an earlier
|
||||
// release made, is refused, said (H-A-483, third addendum): its consumer would be another holder's old durable.
|
||||
func (pq *publishQueue) learn(ctx context.Context, holder string) {
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
if strings.HasSuffix(holder, "-"+class) {
|
||||
pq.logf("the publishes of %s are refused: a holder is a module name, and one ending in -%s names the class durable of an earlier release (H-A-483)", holder, class)
|
||||
return
|
||||
}
|
||||
}
|
||||
pq.mu.Lock()
|
||||
if pq.pulling == nil {
|
||||
pq.pulling = map[string]bool{}
|
||||
@@ -636,94 +642,71 @@ func (pq *publishQueue) learn(ctx context.Context, holder string) {
|
||||
}
|
||||
}
|
||||
|
||||
// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item
|
||||
// waits while a live one waits on the queue.
|
||||
// pull makes holder's one consumer and dispatches its items (design 47 H2 §793, H-A-483 and its addenda): first it retires the
|
||||
// holder's class durables an earlier release made (`-live`, `-again`), whose filters a work queue refuses beside the one; their
|
||||
// unacknowledged items stay on the queue and reach the one consumer.
|
||||
func (pq *publishQueue) pull(ctx context.Context, holder string) error {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chans := map[string]chan jetstream.Msg{}
|
||||
var live jetstream.Consumer
|
||||
for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { // the again first: the live is first by the dispatch, not by luck
|
||||
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class),
|
||||
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err)
|
||||
}
|
||||
ch := make(chan jetstream.Msg, publishesMaxAckPending)
|
||||
cc, err := c.Consume(func(m jetstream.Msg) { ch <- m }, jetstream.PullMaxMessages(publishesMaxAckPending))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s is not pulled: %w", publishesConsumer(holder, class), err)
|
||||
}
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
chans[class] = ch
|
||||
if class == versions.PublishClassLive {
|
||||
live = c
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
old := publishesConsumer(holder) + "-" + class
|
||||
err := pq.js.DeleteConsumer(ctx, b.Stream, old)
|
||||
switch {
|
||||
case err == nil:
|
||||
pq.logf("the consumer %s of an earlier release is removed: the publishes of %s have one consumer (H-A-483)", old, holder)
|
||||
case !errors.Is(err, jetstream.ErrConsumerNotFound):
|
||||
return fmt.Errorf("the consumer %s of an earlier release is not removed: %w", old, err)
|
||||
}
|
||||
}
|
||||
liveWaits := func() bool {
|
||||
info, err := live.Info(ctx)
|
||||
return err == nil && info.NumPending > 0
|
||||
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder),
|
||||
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + ".>", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder), err)
|
||||
}
|
||||
pq.logf("pulling the publishes of %s, the live first", holder)
|
||||
go pq.dispatch(ctx, chans[versions.PublishClassLive], chans[versions.PublishClassStart], liveWaits)
|
||||
pq.logf("pulling the publishes of %s", holder)
|
||||
go pq.dispatch(ctx, holder, c)
|
||||
return nil
|
||||
}
|
||||
|
||||
// dispatch runs the items of one holder, at most four at once, every live item before an again one: an again item pulled while
|
||||
// a live one waits on the queue (liveWaits) runs after it.
|
||||
func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstream.Msg, liveWaits func() bool) {
|
||||
// dispatch runs the items of one holder in the order its one consumer gives them, at most four at once (design 47 H2 §793,
|
||||
// H-A-483): each free slot takes the next item through a pull that waits, so an item put or handed back after its pause is
|
||||
// received the moment the bus gives it; an item handed back reappears only after its pause, so an item put meanwhile goes
|
||||
// before it. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of it is put.
|
||||
func (pq *publishQueue) dispatch(ctx context.Context, holder string, c jetstream.Consumer) {
|
||||
sem := make(chan struct{}, versions.PublishSendsAtOnce)
|
||||
run := func(m jetstream.Msg) {
|
||||
go func() {
|
||||
defer func() { <-sem }()
|
||||
pq.take(ctx, m)
|
||||
}()
|
||||
}
|
||||
var spare jetstream.Msg // an again item pulled while a live one waited
|
||||
for {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
select {
|
||||
case m := <-live:
|
||||
run(m)
|
||||
continue
|
||||
default:
|
||||
}
|
||||
m := spare
|
||||
spare = nil
|
||||
if m == nil {
|
||||
select {
|
||||
case lm := <-live:
|
||||
run(lm)
|
||||
continue
|
||||
case m = <-again:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
var got jetstream.Msg
|
||||
b, err := c.Fetch(1)
|
||||
if err == nil {
|
||||
for m := range b.Messages() {
|
||||
got = m
|
||||
}
|
||||
err = b.Error()
|
||||
}
|
||||
if liveWaits() {
|
||||
spare = m
|
||||
select {
|
||||
case lm := <-live:
|
||||
run(lm)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
if err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil {
|
||||
<-sem
|
||||
pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err)
|
||||
pq.mu.Lock()
|
||||
delete(pq.pulling, holder)
|
||||
pq.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if got == nil {
|
||||
<-sem
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case lm := <-live: // came while the queue was asked
|
||||
spare = m
|
||||
run(lm)
|
||||
default:
|
||||
run(m)
|
||||
}
|
||||
go func(m jetstream.Msg) {
|
||||
defer func() { <-sem }()
|
||||
pq.take(ctx, m)
|
||||
}(got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -827,45 +810,6 @@ func (pq *publishQueue) drop(ctx context.Context, items *publishItems, module, v
|
||||
return fmt.Sprintf("the publish of %s %s is dropped: %d items terminated, and none is sent or put again", module, version, len(qs)), nil
|
||||
}
|
||||
|
||||
// first is builder_publish_first (§793): each again item of the version is put again as live, under the message id
|
||||
// `<holder>.live.<module>.<version>`, and the old terminated; a live id the queue holds within its window takes nothing, and the
|
||||
// old stays, said.
|
||||
func (pq *publishQueue) first(ctx context.Context, items *publishItems, module, version string) (string, error) {
|
||||
qs, st, err := pq.itemsOf(ctx, module, version, versions.PublishClassStart)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
|
||||
}
|
||||
if len(qs) == 0 {
|
||||
return "", fmt.Errorf("the work queue %s holds no again item of %s %s", versions.PublishesLocal, module, version)
|
||||
}
|
||||
var moved, kept []string
|
||||
for subj, q := range qs {
|
||||
q.Class = versions.PublishClassLive
|
||||
dup, err := pq.put(ctx, q)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s %s is not put again as live at %s: %w", module, version, q.Holder, err)
|
||||
}
|
||||
if dup {
|
||||
kept = append(kept, q.Holder)
|
||||
continue
|
||||
}
|
||||
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
|
||||
return "", fmt.Errorf("the again item of %s %s at %s is not terminated: %w", module, version, q.Holder, err)
|
||||
}
|
||||
moved = append(moved, q.Holder)
|
||||
}
|
||||
for _, h := range items.settle(module, version, func(h heldItem) bool { return h.q.Class == versions.PublishClassStart }) {
|
||||
_ = h.m.Term()
|
||||
}
|
||||
sort.Strings(moved)
|
||||
answer := fmt.Sprintf("%s %s is put first as live at %s, its again item terminated", module, version, strings.Join(moved, ", "))
|
||||
if len(kept) > 0 {
|
||||
sort.Strings(kept)
|
||||
answer += fmt.Sprintf("; at %s the work queue holds its live id within the duplicates window, so its again item stays", strings.Join(kept, ", "))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// endItem writes the version's publish item ended with answer, made when none stands: Try and take then leave it, and a start
|
||||
// asks it no more.
|
||||
func (p *publishItems) endItem(ctx context.Context, module, version, answer string) error {
|
||||
|
||||
@@ -440,3 +440,17 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-483 (third addendum): a holder is a module name, so one ending in -live or -again, the class durables of an
|
||||
// earlier release, is refused at the learn, said, and nothing of it is pulled.
|
||||
func TestAHolderNamedLikeAClassDurableIsRefused(t *testing.T) {
|
||||
for _, holder := range []string{"oci-live", "oci-again"} {
|
||||
var said []string
|
||||
pq := &publishQueue{logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) },
|
||||
binding: func() (publishesStream, error) { return publishesStream{}, errors.New("no bus in a unit test") }}
|
||||
pq.learn(context.Background(), holder)
|
||||
if len(said) != 1 || !strings.Contains(said[0], "the publishes of "+holder+" are refused") || pq.pulling[holder] {
|
||||
t.Errorf("%s: said %q, pulling %v, want it refused at the learn, said once, nothing pulled", holder, said, pq.pulling[holder])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi
|
||||
released(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
|
||||
go versionActed(ctx, acts, kv, e, r, logf)
|
||||
return
|
||||
@@ -208,11 +208,8 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue
|
||||
r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them"
|
||||
}
|
||||
}
|
||||
} else if r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst {
|
||||
} else if r.Mode == build.ModePublishDrop {
|
||||
act := acts.PublishDrop
|
||||
if r.Mode == build.ModePublishFirst {
|
||||
act = acts.PublishFirst
|
||||
}
|
||||
if act == nil {
|
||||
err = errors.New("this builder keeps no work queue of publishes")
|
||||
} else {
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
@@ -57,8 +56,9 @@ type reservations struct {
|
||||
logf func(string, ...any)
|
||||
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
|
||||
withdraw func(module, fingerprint, why string)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
call toolCaller
|
||||
// recordAgain records the digests of a done build of module at fp its end did not record, saying so (design 47 H2 H-A-492);
|
||||
// nil records none.
|
||||
recordAgain func(ctx context.Context, module, fp, when string) (string, error)
|
||||
// rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none.
|
||||
rebuild func(ctx context.Context, module, fp, why string) error
|
||||
// say says package-published on the seat mesh-builds; nil says none.
|
||||
@@ -117,7 +117,7 @@ func sayReservedOn(js jetstream.JetStream, cred process.Account) func(context.Co
|
||||
// newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's
|
||||
// verb versions through call (design 47 G §348), the only source numbering reads them from.
|
||||
func newReservations(store versions.ReserveStore, call toolCaller, bounds func() jetstream.KeyValue, found *findings, wake func(context.Context), logf func(string, ...any)) *reservations {
|
||||
rv := &reservations{call: call, bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf}
|
||||
rv := &reservations{bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf}
|
||||
rv.r = &versions.Reserver{Store: store, Filled: rv.filled.Load, ReleasedOf: func(_ context.Context, module string) ([]string, error) {
|
||||
raw, err := call("seat:"+meshcatalog.Seat+".versions", meshcatalog.VersionsInput{Module: module})
|
||||
if err != nil {
|
||||
@@ -134,8 +134,6 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func()
|
||||
return out, nil
|
||||
}}
|
||||
rv.r.Took, rv.r.Logf, rv.r.Reserved = rv.took, logf, rv.reservedSaid
|
||||
// A module's tags are read before it reserves, and a number tagged at another commit skipped (design 47 H2 §796).
|
||||
rv.r.Tags, rv.r.TaggedElsewhere = tagsVia(call), rv.taggedElsewhere
|
||||
return rv
|
||||
}
|
||||
|
||||
@@ -310,7 +308,7 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err)
|
||||
break
|
||||
}
|
||||
if err := rv.tagged(ctx, ev); err != nil {
|
||||
if err := rv.release(ctx, ev); err != nil {
|
||||
rv.logf("%v; heard again", err)
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
@@ -318,24 +316,6 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf)
|
||||
return true
|
||||
}
|
||||
gone, err := rv.r.Released(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
rv.logf("%v; heard again", err)
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf)
|
||||
return true
|
||||
}
|
||||
for _, k := range gone {
|
||||
rv.dropBound(ctx, k)
|
||||
}
|
||||
if rv.released != nil {
|
||||
rv.released(ctx, ev.Module, ev.Version)
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@@ -345,6 +325,87 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// release takes a released version in as the catalog's version-released says it (design 47 G §369): its reservation released,
|
||||
// its bound and publish item ended and its findings cleared. No git tag is written (design 47 H2 H-A-494).
|
||||
func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error {
|
||||
gone, err := rv.r.Released(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range gone {
|
||||
rv.dropBound(ctx, k)
|
||||
}
|
||||
if rv.released != nil {
|
||||
rv.released(ctx, ev.Module, ev.Version)
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) // one raised before H-A-494
|
||||
return nil
|
||||
}
|
||||
|
||||
// releasedAtStart takes in, at the builder's start, every built, unreleased reservation whose version the catalog's verb versions
|
||||
// holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one
|
||||
// said while the builder was away, is taken all the same, its reservation released and its publish item ended; no tag is written
|
||||
// (design 47 H2 H-A-494).
|
||||
// A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released.
|
||||
// Before that, a standing reservation with no digests recorded whose build key is done has them recorded from the done key
|
||||
// (design 47 H2 H-A-492), so it is held against the released versions too.
|
||||
func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
all, err := rv.r.All(ctx)
|
||||
if err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
if rv.recordedAtStart(ctx, all) {
|
||||
if all, err = rv.r.All(ctx); err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
read := map[string][]string{}
|
||||
for _, v := range builtOf(all) {
|
||||
if versions.NotStanding(v, true) != "" {
|
||||
continue
|
||||
}
|
||||
released, ok := read[v.Module]
|
||||
if !ok {
|
||||
if released, err = rv.r.ReleasedOf(ctx, v.Module); err != nil {
|
||||
rv.logf("%s: the catalog's released versions cannot be read at the start, so its built reservations wait for its next version-released: %v", v.Module, err)
|
||||
}
|
||||
read[v.Module] = released
|
||||
}
|
||||
if !slices.Contains(released, v.Version) {
|
||||
continue
|
||||
}
|
||||
if err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil {
|
||||
rv.logf("%s %s is released in the catalog and not taken in at the start: %v", v.Module, v.Version, err)
|
||||
continue
|
||||
}
|
||||
rv.logf("%s %s is released in the catalog: taken in at the start, as its version-released", v.Module, v.Version)
|
||||
}
|
||||
}
|
||||
|
||||
// recordedAtStart records the digests of every standing reservation that has none recorded and whose build key is done (design 47
|
||||
// H2 H-A-492): a record its build's end lost. It answers whether one was recorded; one that cannot be is said.
|
||||
func (rv *reservations) recordedAtStart(ctx context.Context, all []versions.ReservedVersion) bool {
|
||||
if rv.recordAgain == nil {
|
||||
return false
|
||||
}
|
||||
recorded := false
|
||||
for _, v := range all {
|
||||
if !v.BuiltAt.IsZero() || v.Fingerprint == "" || versions.NotStanding(v, true) != "" {
|
||||
continue
|
||||
}
|
||||
version, err := rv.recordAgain(ctx, v.Module, v.Fingerprint, "at the builder's start")
|
||||
if err != nil {
|
||||
rv.logf("%v", err)
|
||||
}
|
||||
recorded = recorded || version != ""
|
||||
}
|
||||
return recorded
|
||||
}
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told
|
||||
// before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told
|
||||
// nothing.
|
||||
@@ -404,33 +465,6 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str
|
||||
return fmt.Sprintf("the reservation of %s %s is dropped (%s, %s)", module, series, by, why), nil
|
||||
}
|
||||
|
||||
// tagged writes the git tag of a released version (design 47 G §370): on the catalog's version-released, through the seat
|
||||
// git's verb tag, at the commit the reservation of that version was made from. A version with no reservation (a superseded
|
||||
// build's version is never released, and the key of a version released by another builder is gone) or one that names no
|
||||
// tag (a module with no Go package) is tagged nowhere. A tag the forge could not write is an error, the message heard again.
|
||||
func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error {
|
||||
res, found, err := rv.r.Of(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the reservation of %s %s cannot be read to tag it: %w", ev.Module, ev.Version, err)
|
||||
}
|
||||
if !found || res.Tag == "" || res.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
raw, err := rv.call("seat:"+git.Seat+".tag", git.TagInput{Repository: res.Repository, Commit: res.Commit, Tag: res.Tag})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the tag %s of %s is not written: %w", res.Tag, ev.Module, err)
|
||||
}
|
||||
var a git.TagAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return fmt.Errorf("the git seat answered what is no tag answer for %s: %w", res.Tag, err)
|
||||
}
|
||||
if a.Error != "" || !a.Performed {
|
||||
return fmt.Errorf("the tag %s of %s is not written: %s (%s)", res.Tag, ev.Module, a.Error, a.ErrorKind)
|
||||
}
|
||||
rv.logf("%s is tagged %s at %s", ev.Module, res.Tag, res.Commit)
|
||||
return nil
|
||||
}
|
||||
|
||||
// noMark says why a published marked nothing, from the reservations the published read (design 47 H H-A-280: no second read):
|
||||
// what each holds of digests, and how many stand at that version by the index by version (its one key, no reservation read).
|
||||
func (rv *reservations) noMark(ctx context.Context, o versions.Outcome, read []versions.ReadReservation) string {
|
||||
@@ -502,19 +536,6 @@ func (rv *reservations) heldElsewhere(ctx context.Context, module, why string) {
|
||||
needs: "builder_reserved shows the reservation and its withdrawn numbers; the registry seat's versions shows what it holds."})
|
||||
}
|
||||
|
||||
// taggedElsewhere raises version-tagged-elsewhere about module (design 47 H2 §796): numbers whose git tag stands at another
|
||||
// commit were skipped at reserving; a tag is never moved. Cleared at the module's next version-released.
|
||||
func (rv *reservations) taggedElsewhere(ctx context.Context, module string, skipped []string) {
|
||||
why := "the tags of " + strings.Join(skipped, ", ") + " stand at another commit, so the reserve skipped them"
|
||||
rv.logf("%s: %s", module, why)
|
||||
rv.found.raise(ctx, finding{kind: versions.VersionTaggedElsewhere, scope: meshconditionsclient.ScopeModule, id: module,
|
||||
summary: module + ": " + why,
|
||||
headline: "A number whose tag stands at another commit is not used",
|
||||
explanation: "The repository holds a tag of " + module + " at a commit other than the build's: " + why + ". A tag is never moved.",
|
||||
resolved: "The catalog says version-released for " + module + ".",
|
||||
needs: "builder_reserved shows the reservation; the seat git's tags shows each tag with its commit."})
|
||||
}
|
||||
|
||||
// movedElsewhere takes a done build whose number a registry holds at another digest off that number (design 47 H H-A-275,
|
||||
// H-A-279): every artifact carries its version from its build, so no digest survives a move. The number is withdrawn as a gap
|
||||
// (H-A-226), the finding raised naming both digests, and the build of the same fingerprint asked again (the build-again path of a
|
||||
|
||||
@@ -156,89 +156,117 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom: design 47 G §370 — the git tag go/vMAJOR.MINOR.PATCH is written through
|
||||
// the seat git's tag on the catalog's version-released, at the commit the version's reservation was made from; a module with no
|
||||
// Go package, and a version never released (a superseded build's), are tagged nowhere; a tag the forge did not write leaves
|
||||
// version-released to be heard again with the reservation standing.
|
||||
func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
// TestAReleasedVersionWritesNoTag: design 47 H2 H-A-494 — the builder writes no git tag at a release, and numbers a series from the
|
||||
// catalog's versions and the registries' alone: the catalog's version-released releases the reservation with no call of the seat
|
||||
// git, a reservation written before (one naming a tag) included; the start's take-in of a version the catalog holds released
|
||||
// (H-A-484 c) the same; and a reserve reads no git tag, a tag standing at another commit no reason to skip a number.
|
||||
func TestAReleasedVersionWritesNoTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var tags []git.TagInput
|
||||
refuse := ""
|
||||
var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796)
|
||||
var gitCalls []string // every call of the seat git: none is made
|
||||
var catalog []meshcatalog.VersionEntry // the released versions the catalog's verb versions answers
|
||||
call := func(addr string, body any) (json.RawMessage, error) {
|
||||
switch addr {
|
||||
case "seat:git.tags":
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true})
|
||||
case "seat:git.tag":
|
||||
default:
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{})
|
||||
if strings.HasPrefix(addr, "seat:git.") {
|
||||
gitCalls = append(gitCalls, addr)
|
||||
if addr == "seat:git.tags" { // a tag standing at another commit, as the forge may still hold one
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}}})
|
||||
}
|
||||
return json.Marshal(git.TagAnswer{Performed: true})
|
||||
}
|
||||
tags = append(tags, body.(git.TagInput))
|
||||
return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse})
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog})
|
||||
}
|
||||
fw := &memFindings{vals: map[string][]byte{}}
|
||||
found := &findings{module: builds.BuilderModule, w: fw, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
store := &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
rv := newReservations(store, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
// Design 47 H2 §796: the tag names the Go module's directory, read from its module path: mesh-sdk's `go` here.
|
||||
rv.r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/go", artifact == "lib" || artifact == "pkg", nil
|
||||
rv.names = func(_ context.Context, _, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/go", artifact == "lib", nil
|
||||
}
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true, GoArtifact: "lib"}
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rv.Reserve(ctx, "prog", "2.0", []string{"prog:bin"}, versions.Source{Repository: "o/p", Commit: "beef"}); err != nil {
|
||||
t.Fatal(err)
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", Fingerprint: "fpA"}
|
||||
// oldRecord rewrites a reservation as one written before H-A-494, naming the tag its release was to be written as.
|
||||
oldRecord := func(module, series, tag string) {
|
||||
key := versions.ReservedKey(module, series)
|
||||
var v versions.ReservedVersion
|
||||
if err := json.Unmarshal(store.vals[key], &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.Tag = tag
|
||||
store.vals[key], _ = json.Marshal(v)
|
||||
}
|
||||
release := func(module, version string) *wakeMsg {
|
||||
m := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"` + module + `","version":"` + version + `","source":1,"artifacts":[]}`)}
|
||||
rv.heard(ctx, m)
|
||||
return m
|
||||
}
|
||||
if m := release("prog", "2.0.0"); !m.acked || len(tags) != 0 {
|
||||
t.Fatalf("a module with no Go package: acked %v, tags %v; want none written", m.acked, tags)
|
||||
// The release path: a Go package's version-released releases it, acknowledged, with no tag written.
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil || v.Tag != "" {
|
||||
t.Fatalf("reserved %+v (%v), want a reservation naming no tag", v, err)
|
||||
}
|
||||
refuse = "forge down"
|
||||
if m := release("pkg", "0.1.0"); !m.naked || m.acked {
|
||||
t.Fatalf("a tag the forge did not write: naked %v acked %v; want the release heard again", m.naked, m.acked)
|
||||
oldRecord("pkg", "0.1", "go/v0.1.0")
|
||||
if m := release("pkg", "0.1.0"); !m.acked || m.naked {
|
||||
t.Fatalf("the release: acked %v naked %v; want it acknowledged", m.acked, m.naked)
|
||||
}
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("the reservation went with an unwritten tag: %v", all)
|
||||
if len(gitCalls) != 0 {
|
||||
t.Errorf("the release called the seat git: %v; want no tag written", gitCalls)
|
||||
gitCalls = nil
|
||||
}
|
||||
refuse = ""
|
||||
tags = nil
|
||||
if m := release("pkg", "0.1.0"); !m.acked || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) {
|
||||
t.Fatalf("the release: acked %v, tags %+v; want go/v0.1.0 at c0ffee of o/r", m.acked, tags)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 {
|
||||
t.Fatalf("the released version still stands: %v", all)
|
||||
}
|
||||
tags = nil
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // build A: 0.1.0 (the stub catalog holds none released)
|
||||
// The start check (H-A-484 c): a built reservation the catalog holds released is released at the start, no tag written; one
|
||||
// not built, or not released, stands.
|
||||
// Design 47 H2 H-A-492: at the start, a standing reservation with no digests recorded whose build key is done has them
|
||||
// recorded (the builder's RecordAgain), and is then held against the released versions as a built one is.
|
||||
if _, err := rv.Reserve(ctx, "rec", "0.1", []string{"rec:bin"}, versions.Source{Repository: "o/q", Commit: "abc", Fingerprint: "fpR"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true, GoArtifact: "lib"}); err != nil || v.Version != "0.1.1" { // build B supersedes A
|
||||
t.Fatalf("second build reserved %q (%v)", v.Version, err)
|
||||
var recordedAt []string
|
||||
rv.recordAgain = func(ctx context.Context, module, fp, when string) (string, error) {
|
||||
recordedAt = append(recordedAt, module+" "+fp+" "+when)
|
||||
_, err := rv.r.Built(ctx, module, "0.1.0", "rec:bin", "sha256:bb", nil) // its done key's digest
|
||||
return "0.1.0", err
|
||||
}
|
||||
release("pkg", "0.1.0") // A's version is never released in fact; were it said, it is no reservation of the key
|
||||
if len(tags) != 0 {
|
||||
t.Fatalf("a superseded build's version was tagged: %+v", tags)
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "rec", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if _, standing, _ := rv.r.Of(ctx, "rec", "0.1.0"); standing || !slices.Equal(recordedAt, []string{"rec fpR at the builder's start"}) {
|
||||
t.Fatalf("at the start, rec 0.1.0 unrecorded and released: standing %v, recorded %q; want it recorded once at the start and released", standing, recordedAt)
|
||||
}
|
||||
if release("pkg", "0.1.1"); len(tags) != 1 || tags[0].Commit != "b0b" {
|
||||
t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags)
|
||||
catalog, rv.recordAgain = nil, nil
|
||||
var ended []string
|
||||
rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) }
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", Fingerprint: "fpB"}); err != nil {
|
||||
t.Fatal(err) // 0.1.0 again: the stub catalog holds none released
|
||||
}
|
||||
// Design 47 H2 §796: a tag standing at another commit makes the next number and the finding version-tagged-elsewhere, which
|
||||
// clears at the module's next version-released.
|
||||
standing = []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}, {Tag: "go/v0.2.1", Commit: "elsewhere"}}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.2" || v.Tag != "go/v0.2.2" {
|
||||
t.Fatalf("reserved %+v (%v) with 0.2.0 and 0.2.1 tagged elsewhere, want 0.2.2 tagged go/v0.2.2", v, err)
|
||||
oldRecord("pkg", "0.1", "go/v0.1.0")
|
||||
rv.releasedAtStart(ctx) // built nothing yet: nothing taken
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("at the start with no build done: reservations %v; want the reservation standing", all)
|
||||
}
|
||||
if _, err := rv.r.Built(ctx, "pkg", "0.1.0", "pkg:lib", "sha256:aa", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.releasedAtStart(ctx) // the catalog holds none released: it stands
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("at the start with nothing released: reservations %v; want the reservation standing", all)
|
||||
}
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "pkg", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 || !slices.Equal(ended, []string{"pkg 0.1.0"}) {
|
||||
t.Fatalf("at the start with 0.1.0 released: reservations %v, items ended %v; want it released and its item ended", all, ended)
|
||||
}
|
||||
if len(gitCalls) != 0 {
|
||||
t.Errorf("the start check called the seat git: %v; want no tag written", gitCalls)
|
||||
gitCalls = nil
|
||||
}
|
||||
catalog, rv.released = nil, nil
|
||||
// Numbering reads no tag: the seat git would answer go/v0.2.0 at another commit, and 0.2.0 is reserved all the same.
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.0" || v.Tag != "" || len(gitCalls) != 0 {
|
||||
t.Errorf("reserved %+v (%v), git calls %v; want 0.2.0 naming no tag, no git tag read", v, err, gitCalls)
|
||||
}
|
||||
key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "pkg", versions.VersionTaggedElsewhere)
|
||||
if got, err := decodeFinding(fw.vals[key]); err != nil || !strings.Contains(got.Summary, "0.2.0") || !strings.Contains(got.Summary, "0.2.1") {
|
||||
t.Fatalf("the finding %s = %+v (%v), want it naming 0.2.0 and 0.2.1", key, got, err)
|
||||
if fw.vals[key] != nil {
|
||||
t.Errorf("a reserve raised %s", key)
|
||||
}
|
||||
if release("pkg", "0.2.2"); fw.vals[key] != nil {
|
||||
t.Errorf("version-released of pkg left %s standing", key)
|
||||
}
|
||||
standing = nil
|
||||
// Design 47 H2 §805, H-A-420 (2): a package's number newly reserved is said as version-reserved, its path the package's name
|
||||
// and its revision the indexer's `<branch>.<n>`; a reservation kept as it stood, an artifact that is no package and a
|
||||
// reservation naming no catalog revision say none.
|
||||
@@ -247,7 +275,7 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
return "git.example/o/r/" + artifact, module == "lib2" && artifact == "pkg", nil
|
||||
}
|
||||
rv.sayReserved = func(_ context.Context, ev meshbuilds.VersionReservedEvent) error { said = append(said, ev); return nil }
|
||||
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", GoPackage: true, GoArtifact: "pkg", Branch: "main", CatalogRevision: 158}
|
||||
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", Branch: "main", CatalogRevision: 158}
|
||||
for range 2 {
|
||||
if _, err := rv.Reserve(ctx, "lib2", "0.3", []string{"lib2:pkg", "lib2:bin"}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -19,6 +19,8 @@ import (
|
||||
// `module` alone the module's keys newest first, at most MostBuilds, each with its fingerprint whole, its state, revision and
|
||||
// since when, its step and node, its error whole, event-builds and the open ask (§568), and `newest` naming each watched branch
|
||||
// whose newest catalog graph entry of the module names that fingerprint, read from the catalog half by the tools bundle.
|
||||
// **Each key's time is its entry's own write time** (design 47 H2 H-A-487): the history of a key is read only for the one key a
|
||||
// fingerprint names, when its entry does not say when it was written, and a listing cut at MostBuilds says so in its note.
|
||||
|
||||
// BuildTool is the builder's read tool of a module's build keys.
|
||||
const BuildTool = "builder_build"
|
||||
@@ -56,14 +58,16 @@ type BuildAnswer struct {
|
||||
Builds []BuildRow `json:"builds"`
|
||||
// Newest is the fingerprint of the module's newest catalog graph entry on each watched branch.
|
||||
Newest map[string]string `json:"newest-entries,omitempty"`
|
||||
// Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint.
|
||||
// Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint, and that the listing
|
||||
// was cut at MostBuilds (H-A-487).
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// BuildReader reads what builder_build answers.
|
||||
type BuildReader struct {
|
||||
Builds StateRead
|
||||
// Entry is a key's value with its revision and when it was written; nil leaves both out.
|
||||
// Entry is a key's value with its revision and when it was written, the entry's own write time (H-A-487); nil leaves both
|
||||
// out. Builds.Written, the key's history, is read only for the key a fingerprint names when Entry says no time.
|
||||
Entry func(key string) (json.RawMessage, uint64, time.Time, error)
|
||||
// Asked is the builder's state asked, which holds the asks the operator was asked.
|
||||
Asked StateRead
|
||||
@@ -96,6 +100,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
if raw == nil {
|
||||
continue
|
||||
}
|
||||
if at.IsZero() && fingerprint != "" && r.Builds.Written != nil {
|
||||
at = r.Builds.Written(key) // one key's history, never every key's (H-A-487)
|
||||
}
|
||||
v, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%s holds what is not a build: %w", key, err)
|
||||
@@ -118,7 +125,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
}
|
||||
return rows[i].Key > rows[j].Key
|
||||
})
|
||||
var cut string
|
||||
if len(rows) > MostBuilds {
|
||||
cut = fmt.Sprintf("%d of %s's %d build keys are answered, the newest by write time; name a fingerprint for another", MostBuilds, module, len(rows))
|
||||
rows = rows[:MostBuilds]
|
||||
}
|
||||
asks := r.openAsks()
|
||||
@@ -149,6 +158,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
out.Note = "no build key of " + module + " is the newest entry's: " + strings.Join(says, "; ")
|
||||
}
|
||||
}
|
||||
if cut != "" {
|
||||
out.Note = strings.TrimPrefix(out.Note+"; "+cut, "; ")
|
||||
}
|
||||
out.Builds = append(out.Builds, rows...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -66,6 +67,31 @@ func TestBuilderBuildAnswersAModulesKeysWhole(t *testing.T) {
|
||||
if n, err := r.Build("m", ""); err != nil || !strings.Contains(n.Note, "no build key of m is the newest entry's") || !strings.Contains(n.Note, "main names src2:"+strings.Repeat("c", 64)) {
|
||||
t.Errorf("a module none of whose keys the newest entry names: %+v, %v", n, err)
|
||||
}
|
||||
|
||||
// H-A-487: a module of 100 keys is read by each entry's own write time, no key's history read, and the listing is cut at the
|
||||
// 20 newest, saying so; a fingerprint's key with no time in its entry has its one history read.
|
||||
many, at := map[string][]byte{}, map[string]time.Time{}
|
||||
base := time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC)
|
||||
for i := range 100 {
|
||||
k, _ := builds.BuilderBuildKey("big", fmt.Sprintf("src2:%064x", i))
|
||||
many[k], at[k] = done, base.Add(time.Duration(i)*time.Minute)
|
||||
}
|
||||
histories := 0
|
||||
st := stateOf(many)
|
||||
st.Written = func(string) time.Time { histories++; return base }
|
||||
big := &BuildReader{Builds: st, Entry: func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, at[k], nil }}
|
||||
a, err = big.Build("big", "")
|
||||
if err != nil || len(a.Builds) != MostBuilds || !a.Builds[0].Since.Equal(base.Add(99*time.Minute)) || histories != 0 ||
|
||||
!strings.Contains(a.Note, "20 of big's 100 build keys are answered") {
|
||||
t.Fatalf("100 keys: %d rows, newest %v, %d histories read, note %q; want the 20 newest, no history, the cut said", len(a.Builds), a.Builds[0].Since, histories, a.Note)
|
||||
}
|
||||
big.Entry = func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, time.Time{}, nil }
|
||||
if a, err := big.Build("big", ""); err != nil || len(a.Builds) != MostBuilds || histories != 0 {
|
||||
t.Fatalf("100 keys with no time in their entries: %d rows (%v), %d histories read; want no history read for a listing", len(a.Builds), err, histories)
|
||||
}
|
||||
if one, err := big.Build("big", fmt.Sprintf("src2:%064x", 7)); err != nil || len(one.Builds) != 1 || !one.Builds[0].Since.Equal(base) || histories != 1 {
|
||||
t.Errorf("one fingerprint with no time in its entry: %+v (%v), %d histories; want its one history read", one, err, histories)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §589: builder_stuck answers each key's fingerprint whole, as the build-input fingerprint the catalog graph entry carries it.
|
||||
|
||||
@@ -170,6 +170,7 @@ func Unreferenced(b builds.BuilderBuild) bool {
|
||||
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
|
||||
// which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
|
||||
// another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder);
|
||||
// - a failed key withdrawn (withdrawnPrefix) is final for its fingerprint, taken again only by a person's build again (H-A-489);
|
||||
// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is
|
||||
// published (ClaimOptions.Published, ADR 0500 clause 6), and not before;
|
||||
// - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its
|
||||
@@ -229,6 +230,8 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)):
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, withdrawnPrefix) && !opt.ByHand:
|
||||
return nil, nil // withdrawn is final for its fingerprint: no look builds it again (H-A-489)
|
||||
case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver && !opt.ByEvent:
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
|
||||
|
||||
@@ -106,11 +106,19 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
|
||||
return nil
|
||||
}
|
||||
raw, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
if err != nil {
|
||||
b.logf("%s: the build key cannot be read, so it is taken as not done: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
cur, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil || cur.State != builds.BuildDone {
|
||||
if err != nil {
|
||||
b.logf("%s: the build key holds what is no build, so it is taken as not done: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if cur.State != builds.BuildDone {
|
||||
return nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
@@ -122,6 +130,57 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
|
||||
return out
|
||||
}
|
||||
|
||||
// DefaultRecordBound is the bound of the record of a reservation's digests, its setting record-bound-seconds' default (design 47
|
||||
// H2 H-A-492).
|
||||
const DefaultRecordBound = time.Minute
|
||||
|
||||
// recordContext is the context a reservation's digests are recorded under: ctx's values with no cancel of its own, bounded by
|
||||
// RecordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record.
|
||||
func (b *Builder) recordContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
bound := b.RecordBound
|
||||
if bound <= 0 {
|
||||
bound = DefaultRecordBound
|
||||
}
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), bound)
|
||||
}
|
||||
|
||||
// RecordAgain records the digests of fp's done build under its standing reservation when its end did not record them (design 47
|
||||
// H2 H-A-492): at the builder's look of the key and at its start (when says which), said each time. It answers the version
|
||||
// recorded, "" when nothing was to be recorded: the key is not done, no reservation stands for fp, or it is recorded already.
|
||||
func (b *Builder) RecordAgain(ctx context.Context, module, fp, when string) (string, error) {
|
||||
b.init()
|
||||
if b.Reserved == nil {
|
||||
return "", nil
|
||||
}
|
||||
digests := b.doneDigests(ctx, module, fp)
|
||||
if digests == nil {
|
||||
return "", nil
|
||||
}
|
||||
res, ok, err := b.Reserved.ReservedFor(ctx, module, fp)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s at %s: whether its done build's digests are recorded cannot be read %s: %w", module, fp, when, err)
|
||||
}
|
||||
if !ok || !res.BuiltAt.IsZero() || len(res.Artifacts) == 0 || versions.NotStanding(res, true) != "" {
|
||||
return "", nil
|
||||
}
|
||||
refs := map[string]string{}
|
||||
for ref := range res.Artifacts {
|
||||
_, artifact, _ := strings.Cut(ref, ":")
|
||||
d, have := digests[artifact]
|
||||
if !have {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded, and the done key holds no digest of %s to record %s", module, fp, res.Version, ref, when)
|
||||
}
|
||||
refs[ref] = d
|
||||
}
|
||||
rec, stop := b.recordContext(ctx)
|
||||
defer stop()
|
||||
if err := b.Reserved.Built(rec, module, res.Version, refs, nil); err != nil {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded %s either: %w", module, fp, res.Version, when, err)
|
||||
}
|
||||
b.logf("%s at %s: the digests of %s were not recorded at its build's end: recorded %s from the done key", module, fp, res.Version, when)
|
||||
return res.Version, nil
|
||||
}
|
||||
|
||||
// PackageNamed looks at a module again when the indexer's packages state names a name for one of its packages
|
||||
// (versions.NamedModule): a done trunk build that left the package unnumbered, its name not known at its claim, is
|
||||
// numbered then with the done build's digests, and version.numbered says them for its build module's late publish.
|
||||
@@ -199,8 +258,8 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
|
||||
}
|
||||
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
|
||||
for _, t := range plan.Tasks {
|
||||
if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage {
|
||||
src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796)
|
||||
if t.Kind == catalogue.ArtifactUpstream {
|
||||
src.Upstream = append(src.Upstream, module+":"+t.Artifact) // copied by its pin, never awaited (H-A-486)
|
||||
}
|
||||
}
|
||||
res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs, src)
|
||||
|
||||
@@ -32,6 +32,8 @@ type fakeReservations struct {
|
||||
next string // the version Reserve answers when set, else the series' .5
|
||||
fail error // the reservation cannot be made (the catalog's versions cannot be read)
|
||||
tookBy map[string]versions.ReservedVersion // by fingerprint, the reservation that took its series over (H-A-244)
|
||||
held map[string]versions.ReservedVersion // by fingerprint, the reservation ReservedFor answers as it stands (H-A-492)
|
||||
lost error // the record of the digests fails, as a store refusing it would
|
||||
}
|
||||
|
||||
func (f *fakeReservations) TakenOver(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
@@ -46,6 +48,9 @@ func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, e
|
||||
func (f *fakeReservations) Filled() bool { return f.filled }
|
||||
|
||||
func (f *fakeReservations) ReservedFor(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
if v, ok := f.held[fp]; ok {
|
||||
return v, true, nil
|
||||
}
|
||||
for _, s := range f.sources {
|
||||
if s.Fingerprint == fp {
|
||||
return versions.ReservedVersion{Version: "held"}, true, nil
|
||||
@@ -65,7 +70,13 @@ func (f *fakeReservations) Reserve(_ context.Context, module, series string, art
|
||||
return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
func (f *fakeReservations) Built(ctx context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err // the state's write runs under ctx: a cancelled one records nothing
|
||||
}
|
||||
if f.lost != nil {
|
||||
return f.lost
|
||||
}
|
||||
f.deps = deps
|
||||
f.built = map[string]string{version: ""}
|
||||
for ref, d := range digests {
|
||||
@@ -118,6 +129,13 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under the reserved version", res.built)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved for the record and named as upstream, so its publish is never awaited.
|
||||
up := &fakeReservations{filled: true}
|
||||
plan := Plan{Series: "0.1", Tasks: []Task{{Artifact: "server", Kind: catalogue.ArtifactUpstream}, {Artifact: "tools", Kind: catalogue.ArtifactBundle}}}
|
||||
if _, ok := (&Builder{Reserved: up}).reserve(context.Background(), "nats", meshcatalog.ModulesByModuleValue{Fp2: "f"}, plan, Origin{}, false); !ok ||
|
||||
!slices.Equal(up.reserved[0], []string{"nats:server", "nats:tools"}) || !slices.Equal(up.sources[0].Upstream, []string{"nats:server"}) {
|
||||
t.Errorf("an upstream artifact reserved %v from %+v, want both refs reserved and nats:server named upstream", up.reserved, up.sources)
|
||||
}
|
||||
r.consider(t, "modules/m")
|
||||
if len(res.reserved) != 1 {
|
||||
t.Errorf("a done key reserved again: %v", res.reserved)
|
||||
@@ -146,6 +164,97 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound: design 47 H2 H-A-492 — the record of a reservation's digests after a
|
||||
// build ends runs under the builder's own bound, so a build context cancelled at its end still records; a done build whose
|
||||
// digests are not recorded is recorded at the builder's next look, said.
|
||||
func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) {
|
||||
manifest := catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
t.Run("a build's context cancelled at its end still records", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
r.b.Ended = func(context.Context, string) { cancel() } // a takeover or the builder's end, once the key is written
|
||||
r.b.Module(ctx, "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
||||
t.Fatalf("build = %+v, want done", v)
|
||||
}
|
||||
if res.built["m:tool"] != digest {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under 0.1.5 though the build's context ended", res.built)
|
||||
}
|
||||
})
|
||||
t.Run("the record's bound is the setting record-bound-seconds, never the build's", func(t *testing.T) {
|
||||
build, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, c := range []struct {
|
||||
bound, want time.Duration
|
||||
}{{0, DefaultRecordBound}, {5 * time.Second, 5 * time.Second}} {
|
||||
rec, stop := (&Builder{RecordBound: c.bound}).recordContext(build)
|
||||
deadline, has := rec.Deadline()
|
||||
stop()
|
||||
if left := time.Until(deadline); !has || left > c.want || left < c.want-time.Second {
|
||||
t.Errorf("RecordBound %v: the record's context has deadline %v (%v left, %v), want %v left of its own", c.bound, deadline, left, has, c.want)
|
||||
}
|
||||
}
|
||||
rec, stop := (&Builder{}).recordContext(build)
|
||||
defer stop()
|
||||
if rec.Err() != nil {
|
||||
t.Errorf("the record's context ended with the build's: %v", rec.Err())
|
||||
}
|
||||
})
|
||||
t.Run("a build key that cannot be read is said, not taken silently", func(t *testing.T) {
|
||||
var journal []string
|
||||
b := &Builder{Claimer: Claimer{Store: unreadable{newMem()}}, Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }}
|
||||
if d := b.doneDigests(context.Background(), "m", fp1); d != nil {
|
||||
t.Errorf("an unreadable key answered digests %v", d)
|
||||
}
|
||||
if !slices.ContainsFunc(journal, func(l string) bool {
|
||||
return strings.Contains(l, "cannot be read") && strings.Contains(l, "bus timeout")
|
||||
}) {
|
||||
t.Errorf("journal %q, want a line saying the build key cannot be read, with why", journal)
|
||||
}
|
||||
})
|
||||
t.Run("a done key whose digests are not recorded is recorded at the next look", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true, lost: errors.New("context canceled")}
|
||||
r.b.Reserved = res
|
||||
var journal []string
|
||||
r.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
|
||||
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone || res.built != nil {
|
||||
t.Fatalf("build %s recorded %v, want done and nothing recorded", v.State, res.built)
|
||||
}
|
||||
res.lost = nil
|
||||
res.held = map[string]versions.ReservedVersion{fp1: {Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": ""}}}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v at the next look, want the done key's digest of m:tool under 0.1.5", res.built)
|
||||
}
|
||||
if !slices.ContainsFunc(journal, func(l string) bool {
|
||||
return strings.Contains(l, "0.1.5 were not recorded") && strings.Contains(l, "recorded at a look")
|
||||
}) {
|
||||
t.Errorf("journal %q, want a line saying 0.1.5's digests were recorded at a look", journal)
|
||||
}
|
||||
// Recorded once: a reservation whose build ended recorded is left.
|
||||
res.built = nil
|
||||
res.held[fp1] = versions.ReservedVersion{Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": digest}, BuiltAt: time.Unix(1, 0)}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built != nil {
|
||||
t.Errorf("a recorded reservation was recorded again: %v", res.built)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// unreadable is a builds bucket whose reads fail, as one on a bus that does not answer.
|
||||
type unreadable struct{ *memStore }
|
||||
|
||||
func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) {
|
||||
return nil, 0, false, errors.New("bus timeout")
|
||||
}
|
||||
|
||||
// TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through
|
||||
// reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why.
|
||||
func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) {
|
||||
@@ -236,7 +345,7 @@ func TestAReservedBuildKeepsWhatItsStepResolvedAndWasBuiltWith(t *testing.T) {
|
||||
|
||||
// TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion: design 47 G §370 — a module whose one artifact is
|
||||
// a Go package (mesh-wireguard's go-overlay) is planned on the Go build seat, its ask carries the reserved version and names
|
||||
// the seat go-package-registry the package is published and tagged through, and the build is done; a build reserving none
|
||||
// the seat go-package-registry the package is published through, and the build is done; a build reserving none
|
||||
// names no seat.
|
||||
func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *testing.T) {
|
||||
goPackage := catalogue.Artifact{Name: "go-overlay", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}}
|
||||
@@ -265,10 +374,6 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t
|
||||
len(asked[0].Build.Artifacts) != 1 || asked[0].Build.Artifacts[0] != (buildstep.AskArtifact{Artifact: "go-overlay", Seat: c.seat}) {
|
||||
t.Fatalf("asks %+v, want one Go ask with version %q naming go-overlay on seat %q", asked, c.version, c.seat)
|
||||
}
|
||||
// Design 47 H2 §796: the reservation names the Go package's artifact, whose module path names its tag's directory.
|
||||
if c.version != "" && (len(res.sources) != 1 || !res.sources[0].GoPackage || res.sources[0].GoArtifact != "go-overlay") {
|
||||
t.Errorf("reserved from %+v, want one source naming the Go package go-overlay", res.sources)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +107,9 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun,
|
||||
}
|
||||
}
|
||||
// Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture
|
||||
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded.
|
||||
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded. Once the picture
|
||||
// heard a newer entry of the module on the first branch, H-A-489 overrides H-A-205: the older fingerprint is refused
|
||||
// superseded (buildEntry), never built or reserved.
|
||||
b.mu.Lock()
|
||||
for _, e := range run.Entries {
|
||||
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
|
||||
|
||||
@@ -127,6 +127,9 @@ type Builder struct {
|
||||
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished)
|
||||
// gives a pin (H-A-192).
|
||||
PackageWait time.Duration
|
||||
// RecordBound bounds the record of a reservation's digests after a build ends, its setting record-bound-seconds (design 47
|
||||
// H2 H-A-492): the builder's own bound, never the build's context; DefaultRecordBound when 0.
|
||||
RecordBound time.Duration
|
||||
// PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first
|
||||
// (H-A-194): read once before a build waits for that package. Nil reads none.
|
||||
PublishedVersions func(ctx context.Context, path string) ([]string, error)
|
||||
@@ -177,6 +180,8 @@ type Builder struct {
|
||||
// publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished):
|
||||
// a package with no pin waits for one (H-A-192).
|
||||
publishedPins map[string]map[string]string
|
||||
// supersededSaid are the build keys refused as superseded (H-A-489), so each is said once.
|
||||
supersededSaid sync.Map
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -310,11 +315,49 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
|
||||
if superseded != nil {
|
||||
// A newer fingerprint of the module: the build of the older one in flight is taken out, and its asks are withdrawn
|
||||
// from the build seats (ADR 0664 §4).
|
||||
superseded(buildrun.Withdrawn{Why: "the build of " + key + " at " + old.Fp2 + " was superseded by the fingerprint " + m.Fp2})
|
||||
superseded(buildrun.Withdrawn{Why: supersededWhy(key, old.Fp2, m.Fp2)})
|
||||
}
|
||||
b.consider(ctx, key, m)
|
||||
}
|
||||
|
||||
// supersededWhy is the words of a build withdrawn because a newer fingerprint of its module superseded it.
|
||||
func supersededWhy(module, fp, newer string) string {
|
||||
return "the build of " + module + " at " + fp + " was superseded by the fingerprint " + newer
|
||||
}
|
||||
|
||||
// withdrawnPrefix opens the error of a key whose build was withdrawn (H-A-225, ADR 0664 §4). It is final for its fingerprint
|
||||
// (H-A-489): no look builds it again; a person's build again does.
|
||||
const withdrawnPrefix = "withdrawn: "
|
||||
|
||||
// supersededBy is the fingerprint that supersedes fp, a fingerprint of module whose build key is key, when fp is of its
|
||||
// registration's first watched branch and is no longer the newest entry heard on that branch (H-A-489); "" otherwise. A key no
|
||||
// entry named is of the first branch, as originOf has it.
|
||||
func (b *Builder) supersededBy(module, key, fp string) string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
o, ok := b.origin[key]
|
||||
if !ok {
|
||||
o = Origin{First: true}
|
||||
}
|
||||
if !o.First {
|
||||
return ""
|
||||
}
|
||||
newest, heard := b.heard[o.Branch+"\x00"+module]
|
||||
if !heard || newest.Fp2 == "" || newest.Fp2 == fp {
|
||||
return ""
|
||||
}
|
||||
return newest.Fp2
|
||||
}
|
||||
|
||||
// saySuperseded says, once per key, that the build of fp is refused as superseded by newer (H-A-489), and gives the words.
|
||||
func (b *Builder) saySuperseded(module, key, fp, newer string) string {
|
||||
why := "is not built: superseded by " + newer + ", the newest entry heard on its first branch"
|
||||
if _, said := b.supersededSaid.LoadOrStore(key, true); !said {
|
||||
b.logf("%s at %s: %s", module, fp, why)
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
|
||||
// directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is
|
||||
// built against, its packages and needs, and the named paths of its artifacts when every artifact names them.
|
||||
@@ -637,6 +680,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
whyEnded = fmt.Sprintf(format, args...)
|
||||
b.logf("%s at %s: %s", module, m.Fp2, whyEnded)
|
||||
}
|
||||
// A fingerprint no longer the newest entry heard on its first branch is neither reserved for nor built (H-A-489): two
|
||||
// fingerprints of one module never take the series from each other.
|
||||
if newer := b.supersededBy(module, key, m.Fp2); newer != "" {
|
||||
whyEnded = b.saySuperseded(module, key, m.Fp2, newer)
|
||||
return
|
||||
}
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
var noModule *NoBuildModule
|
||||
if errors.As(planErr, &noModule) {
|
||||
@@ -680,6 +729,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
|
||||
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
|
||||
if trunk && b.originOf(key).First && planErr == nil && !nothing && !reserving {
|
||||
// A done build whose digests its end did not record is recorded at the next look (design 47 H2 H-A-492).
|
||||
if _, err := b.RecordAgain(ctx, module, m.Fp2, "at a look"); err != nil {
|
||||
b.logf("%v", err)
|
||||
}
|
||||
}
|
||||
if reserving && b.waitFill(module, m) {
|
||||
end("is not built yet: wait-held, the catalog has not said versions-filled")
|
||||
return
|
||||
@@ -727,7 +782,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
}
|
||||
var reserved versions.ReservedVersion
|
||||
hasReserved := false
|
||||
hasReserved, supersededNow := false, ""
|
||||
if rebuild && cl.Build.State == builds.BuildBuilding {
|
||||
// Reserved as A-490b amended by H-A-225 has it: this fingerprint's standing reservation is kept while it is unreleased
|
||||
// and above the highest the catalog's versions answers, else the next above that highest is taken (A-455); the build's
|
||||
@@ -737,10 +792,16 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
reserving = true
|
||||
} else if reserving && cl.Build.State != builds.BuildDone {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
// Looked at again just before it reserves (H-A-489): a newer entry heard since the claim is not taken over.
|
||||
if supersededNow = b.supersededBy(module, key, m.Fp2); supersededNow == "" {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
}
|
||||
}
|
||||
var result builds.BuilderBuild
|
||||
if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
if supersededNow != "" {
|
||||
// Ended as the newer entry's take-in withdraws a build in flight: final for this fingerprint.
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + supersededWhy(module, m.Fp2, supersededNow)})}
|
||||
} else if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
// A trunk build that reserves is never built without its version: it would end done and publish nothing, and its key would
|
||||
// stay done, never built again (A-490). The key fails with why and is taken again at the next look or by builder_build_again.
|
||||
result = failed("no version is reserved for this build: the builder's journal says why, and it is built again once one can be")
|
||||
@@ -765,7 +826,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
var w buildrun.Withdrawn
|
||||
if ctx.Err() == nil && errors.As(context.Cause(run), &w) {
|
||||
// Taken out, not stopped: the key says so, and the asks are withdrawn (the asker dropped them).
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{"withdrawn: " + w.Why})}
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + w.Why})}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
@@ -807,8 +868,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
for _, a := range result.Artifacts {
|
||||
digests[module+":"+a.Name] = a.Digest
|
||||
}
|
||||
if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err)
|
||||
// Under the builder's own bound, never the build's context: a takeover or an end that cancels it no longer leaves a passed
|
||||
// build unrecorded, so unreleased (design 47 H2 H-A-492).
|
||||
rec, stop := b.recordContext(ctx)
|
||||
err := b.Reserved.Built(rec, module, reserved.Version, digests, b.takeDependencies(key))
|
||||
stop()
|
||||
if err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v; recorded again at the builder's next look or start", cl.Key, reserved.Version, err)
|
||||
}
|
||||
}
|
||||
if result.State == builds.BuildFailed {
|
||||
|
||||
@@ -333,8 +333,9 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
asked, release := make(chan struct{}), make(chan struct{})
|
||||
answer := r.answer
|
||||
var once sync.Once
|
||||
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
||||
close(asked)
|
||||
once.Do(func() { close(asked) })
|
||||
<-release
|
||||
return answer(a)
|
||||
}
|
||||
@@ -356,6 +357,85 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "withdrawn: its reservation 0.1.3 was taken over") {
|
||||
t.Fatalf("the key = %+v, want failed withdrawn naming the take-over", v)
|
||||
}
|
||||
// H-A-489: withdrawn is final for the fingerprint: a later look builds it not again; a person's build again does.
|
||||
n, rev := len(r.asks.Asks()), r.st.revs[r.key]
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); len(r.asks.Asks()) != n || r.st.revs[r.key] != rev || !strings.HasPrefix(v.Error, "withdrawn: ") {
|
||||
t.Errorf("a later look built the withdrawn key again: %d asks (was %d), the key %+v", len(r.asks.Asks()), n, v)
|
||||
}
|
||||
if cl, err := r.b.Claimer.Claim(context.Background(), "m", fp1, mdigest('a'), ClaimOptions{ByHand: true}); err != nil || cl == nil {
|
||||
t.Errorf("a person's build again of the withdrawn key was not claimed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-489 (review round 1): a newer entry of the first branch heard between the older build's claim and its reservation is
|
||||
// not taken over: the older key ends failed withdrawn with no reservation, and the newer reserves.
|
||||
func TestANewerEntryHeardBeforeTheReservationIsNotTakenOver(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
newer, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
Fp2: fp2, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.st.before = func() { r.b.Module(context.Background(), "m", newer) } // at the older key's claim
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, "withdrawn: ") || !strings.Contains(v.Error, "superseded by the fingerprint "+fp2) {
|
||||
t.Errorf("the older key = %+v, want failed withdrawn, superseded by %s", v, fp2)
|
||||
}
|
||||
if len(res.sources) != 1 || res.sources[0].Fingerprint != fp2 {
|
||||
t.Errorf("reserved %+v, want one reservation, the newer's", res.sources)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-489: a fingerprint of the first branch that is no longer its newest entry heard is neither reserved for nor built, and
|
||||
// said once; the again loop that looks at it later builds nothing.
|
||||
func TestAnOlderFingerprintAskedAfterANewerEntryIsRefused(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
r.b.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
r.bm.fails["ace/build_go_build"] = "go build of tool: boom"
|
||||
older := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: fp1, Manifest: mdigest('a'),
|
||||
Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
delete(r.bm.fails, "ace/build_go_build")
|
||||
newer := older
|
||||
newer.Fp2 = fp2
|
||||
data, err := meshcatalogclient.EncodeModule(newer)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
r.b.Wait()
|
||||
rev := r.st.revs[r.key]
|
||||
for range 2 {
|
||||
r.b.consider(context.Background(), "m", older) // as consider's again loop or a step's look takes it
|
||||
r.b.Wait()
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || r.st.revs[r.key] != rev {
|
||||
t.Errorf("the older key = %+v at revision %d (was %d), want it left as it failed", v, r.st.revs[r.key], rev)
|
||||
}
|
||||
k2, _ := builds.BuilderBuildKey("m", fp2)
|
||||
if v := valueOf(t, r.st, k2); v.State != builds.BuildDone {
|
||||
t.Errorf("the newer key = %+v, want done", v)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range said {
|
||||
if strings.Contains(l, fp1+": is not built: superseded by "+fp2) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("superseded said %d times, want once: %q", n, said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key
|
||||
|
||||
@@ -142,6 +142,10 @@ func (b *Builder) buildEntry(ctx context.Context, pipeline string, e meshbuilds.
|
||||
}
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, failedWords, "existing build")
|
||||
}
|
||||
if newer := b.supersededBy(entry.Module, key, entry.Fp); newer != "" {
|
||||
// Refused, not waited on (H-A-489): no look builds a superseded fingerprint, so its end would never come.
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s %s", entry.Module, entry.Fp, b.saySuperseded(entry.Module, key, entry.Fp, newer)), "refused")
|
||||
}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
|
||||
b.consider(ctx, entry.Module, m)
|
||||
@@ -232,7 +236,7 @@ func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed str
|
||||
switch {
|
||||
case v.State == builds.BuildDone && !Unpublished(v, b.Seats):
|
||||
return true, "", true
|
||||
case v.State == builds.BuildFailed && builds.FailedUntilChanged(v.Error):
|
||||
case v.State == builds.BuildFailed && (builds.FailedUntilChanged(v.Error) || strings.HasPrefix(v.Error, withdrawnPrefix)):
|
||||
return false, key + ": " + v.Error, true
|
||||
}
|
||||
return false, "", false
|
||||
|
||||
@@ -74,6 +74,21 @@ func TestABuildStepAnswersItsEnd(t *testing.T) {
|
||||
if end := ask(entries...); end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Step != meshbuilds.StepEndedStepBuild || len(end.Entries) != 1 || end.Entries[0].Commit != "c1" || end.Pipeline != "main.1.m" {
|
||||
t.Errorf("a done build: %+v", end)
|
||||
}
|
||||
// H-A-489 (review round 1): a withdrawn key whose fingerprint is still the newest entry is answered failed at once, naming
|
||||
// withdrawn: no look builds it again, so its end is never waited for.
|
||||
rw := newRig(t, catalogue.Artifact{Name: "files", Kind: "archive"})
|
||||
rw.b.StepEntry = r.b.StepEntry
|
||||
withdrawn, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Manifest: mdigest('a'), Error: withdrawnPrefix + "its reservation 0.1.3 was taken over"})
|
||||
if _, err := rw.st.Create(t.Context(), rw.key, withdrawn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rw.b.Module(t.Context(), "m", moduleValue(t, "modules/m")) // the newest entry heard is fp1
|
||||
rw.b.Wait()
|
||||
bounded, cancel := context.WithTimeout(t.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
if end := rw.b.BuildStep(bounded, "ask1", meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries}); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "withdrawn: ") {
|
||||
t.Errorf("a withdrawn key asked: %+v, want failed at once naming withdrawn", end)
|
||||
}
|
||||
}
|
||||
|
||||
// A step ask waits for the end of its build without a timer: the log entry of that end wakes it (design 47 G A-97).
|
||||
@@ -434,9 +449,10 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved
|
||||
// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs;
|
||||
// a kept revision naming another fingerprint fails the step, naming it.
|
||||
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, while the older pipeline
|
||||
// asks the build of revision 45. Its kept catalog revision is read by number and the build runs while the picture never heard
|
||||
// a newer entry of m (a restart); once it heard one on the first branch, H-A-489 overrides H-A-205 and the older ask is refused
|
||||
// superseded. A kept revision naming another fingerprint fails the step, naming it.
|
||||
func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
var held atomic.Int64
|
||||
held.Store(46)
|
||||
@@ -460,6 +476,24 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 {
|
||||
t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources)
|
||||
}
|
||||
// H-A-489: once the picture heard the newer entry of m on the first branch, the older ask is refused superseded and
|
||||
// nothing is built or reserved for it; the newer entry's build keeps the series.
|
||||
rn, resn, entryn := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
rn.b.RevisionWait = time.Nanosecond
|
||||
rn.b.FirstBranch = r.b.FirstBranch
|
||||
rn.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
return 45, map[string]string{"m": fp1}, nil
|
||||
}
|
||||
entryn.Fp = fp2
|
||||
rn.b.Entry(context.Background(), Origin{Branch: "main", Source: 46, First: true}, entryn)
|
||||
rn.b.Wait()
|
||||
if endn := rn.b.BuildStep(t.Context(), "a1", run); endn.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(endn.Said, "superseded by "+fp2) {
|
||||
t.Errorf("the older ask after the newer entry: end = %+v; want failed, superseded by %s", endn, fp2)
|
||||
}
|
||||
rn.b.Wait()
|
||||
if _, built := rn.st.vals[rn.key]; built || len(resn.sources) != 1 || resn.sources[0].Fingerprint != fp2 {
|
||||
t.Errorf("built %v, reserved %+v; want nothing built for the older, one reservation, the newer's", built, resn.sources)
|
||||
}
|
||||
r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
r2.b.RevisionWait = time.Nanosecond
|
||||
r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
|
||||
@@ -74,14 +74,11 @@ const ModePublish = "publish"
|
||||
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
|
||||
const ModeRepublish = "republish"
|
||||
|
||||
// ModePublishDrop and ModePublishFirst are the Modes of a reasks value of builder_publish_drop and builder_publish_first (design
|
||||
// 47 H2 §793): Module and Version name the item; the builder terminates it, or puts it again as live and terminates the old.
|
||||
const (
|
||||
ModePublishDrop = "publish-drop"
|
||||
ModePublishFirst = "publish-first"
|
||||
)
|
||||
// ModePublishDrop is the Mode of a reasks value of builder_publish_drop (design 47 H2 §793): Module and Version name the item;
|
||||
// the builder terminates it.
|
||||
const ModePublishDrop = "publish-drop"
|
||||
|
||||
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) or builder_publish_first (ModePublishFirst) writes
|
||||
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) writes
|
||||
// (§793): its key, `<mode>.<unix nanoseconds>` (a call is one value), and the value. Module, version and why are required.
|
||||
func NewPublishAct(mode, module, version, why, by string, at time.Time) (string, []byte, error) {
|
||||
module, version = strings.TrimSpace(module), strings.TrimSpace(version)
|
||||
|
||||
@@ -27,10 +27,8 @@ type Acts struct {
|
||||
ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error)
|
||||
// DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not.
|
||||
DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error)
|
||||
// PublishDrop terminates one publish item, and PublishFirst puts it again as live and terminates the old (builder_publish_drop,
|
||||
// builder_publish_first, design 47 H2 §793): the answer, or why not.
|
||||
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
PublishFirst func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
// PublishDrop terminates one publish item (builder_publish_drop, design 47 H2 §793): the answer, or why not.
|
||||
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
}
|
||||
|
||||
// FloorAgain is builder_floor_again of name: refused, since the builder numbers nothing of its own and no claim asks a floor
|
||||
|
||||
@@ -37,9 +37,8 @@ const (
|
||||
PublishQueueTool = "builder_publish_queue"
|
||||
// QueueFull is what version-unreleased says of a version the work queue's cap refused.
|
||||
QueueFull = "queue full"
|
||||
// PublishDropTool and PublishFirstTool are builder_publish_drop and builder_publish_first (§793).
|
||||
PublishDropTool = "builder_publish_drop"
|
||||
PublishFirstTool = "builder_publish_first"
|
||||
// PublishDropTool is builder_publish_drop (§793); builder_publish_first is retired (H-A-483, third addendum).
|
||||
PublishDropTool = "builder_publish_drop"
|
||||
// NotStoredWords opens what a refused artifact is said as.
|
||||
NotStoredWords = "not stored: "
|
||||
// VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds
|
||||
@@ -48,7 +47,7 @@ const (
|
||||
VerbVersions = "versions"
|
||||
// PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on;
|
||||
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled
|
||||
// live first, each class oldest first (design 47 H H-A-290 (1), H2 §793).
|
||||
// by its one consumer in the order they are put (design 47 H H-A-290 (1), H2 §793, H-A-483).
|
||||
PublishClassLive = "live"
|
||||
PublishClassStart = "again"
|
||||
// PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793).
|
||||
@@ -97,21 +96,19 @@ type QueuedPublish struct {
|
||||
Class string `json:"class,omitempty"`
|
||||
}
|
||||
|
||||
// rank is the place of q's class in a holder's queue: the live first.
|
||||
func (q QueuedPublish) rank() int {
|
||||
if q.Class == PublishClassLive {
|
||||
return 0
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// sortQueued sorts a holder's queued versions in the order they are sent: the live class first, each class oldest first.
|
||||
func sortQueued(qs []QueuedPublish) {
|
||||
sort.SliceStable(qs, func(i, j int) bool {
|
||||
if a, b := qs[i].rank(), qs[j].rank(); a != b {
|
||||
// streamOrder sorts the keys of the work queue's items in stream order: each key is its item's stream sequence (the builder's
|
||||
// reader, publishitem.go), compared as a number; a key that is no number sorts after them by its words.
|
||||
func streamOrder(keys []string) {
|
||||
sort.SliceStable(keys, func(i, j int) bool {
|
||||
a, aerr := strconv.ParseUint(keys[i], 10, 64)
|
||||
b, berr := strconv.ParseUint(keys[j], 10, 64)
|
||||
switch {
|
||||
case aerr == nil && berr == nil:
|
||||
return a < b
|
||||
case aerr == nil || berr == nil:
|
||||
return aerr == nil
|
||||
}
|
||||
return qs[i].Queued.Before(qs[j].Queued)
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
}
|
||||
|
||||
@@ -456,7 +453,7 @@ func (r *Republisher) Republish(ctx context.Context, module, version string) ([]
|
||||
type Queued struct {
|
||||
Total int `json:"total"`
|
||||
Counts []QueueCount `json:"counts"`
|
||||
// Queue are the first queued versions, per holder in the order they are sent (the live class first, each class oldest first);
|
||||
// Queue are the first queued versions, in stream order, the order each holder's one consumer gives them (H-A-483);
|
||||
// with a module named, only that module's.
|
||||
Queue []QueuedPublish `json:"queue"`
|
||||
// Listed says how many of Total the answer lists.
|
||||
@@ -512,7 +509,8 @@ func PublishItemKey(module, version string) string {
|
||||
|
||||
// PublishQueueOf is the seat mesh-builds' verb publish-queue (design 47 H2 §793; H-A-454): the builder's read of its own work queue
|
||||
// publishes (queue, its items read in bulk, capped past its bound), small at any size of the queue (H-A-290 (5)): the items' count
|
||||
// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in the order they are sent; module
|
||||
// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in stream order, the order the
|
||||
// holder's one consumer gives them (H-A-483, third addendum); module
|
||||
// (and version) narrow the list to one module's (one version's), so an item's class can be read.
|
||||
func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQueueAnswer, error) {
|
||||
out := meshbuilds.PublishQueueAnswer{Counts: []meshbuilds.PublishQueueAnswerCountsItem{}, Queue: []meshbuilds.PublishQueueAnswerQueueItem{}}
|
||||
@@ -525,7 +523,13 @@ func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQue
|
||||
}
|
||||
counts := map[[2]string]*meshbuilds.PublishQueueAnswerCountsItem{}
|
||||
var listed []QueuedPublish
|
||||
for _, raw := range vals {
|
||||
keys := make([]string, 0, len(vals))
|
||||
for k := range vals {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
streamOrder(keys)
|
||||
for _, k := range keys {
|
||||
raw := vals[k]
|
||||
var q QueuedPublish
|
||||
if json.Unmarshal(raw, &q) != nil {
|
||||
continue
|
||||
@@ -559,25 +563,12 @@ func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQue
|
||||
}
|
||||
return out.Counts[i].Class < out.Counts[j].Class
|
||||
})
|
||||
sort.SliceStable(listed, func(i, j int) bool { return listed[i].Holder < listed[j].Holder })
|
||||
byHolder := map[string][]QueuedPublish{}
|
||||
var holders []string
|
||||
for _, q := range listed {
|
||||
if byHolder[q.Holder] == nil {
|
||||
holders = append(holders, q.Holder)
|
||||
}
|
||||
byHolder[q.Holder] = append(byHolder[q.Holder], q)
|
||||
}
|
||||
for _, h := range holders {
|
||||
qs := byHolder[h]
|
||||
sortQueued(qs)
|
||||
for _, q := range qs {
|
||||
if len(out.Queue) == PublishQueueListCap {
|
||||
break
|
||||
}
|
||||
out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version,
|
||||
Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued})
|
||||
if len(out.Queue) == PublishQueueListCap {
|
||||
break
|
||||
}
|
||||
out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version,
|
||||
Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued})
|
||||
}
|
||||
out.Listed = int64(len(out.Queue))
|
||||
return out, nil
|
||||
@@ -631,20 +622,17 @@ func AnswerQueue(q meshbuilds.PublishQueueAnswer, acted, reasks Reader) (Queued,
|
||||
|
||||
// PublishQueueDescription is builder_publish_queue's description.
|
||||
const PublishQueueDescription = "The versions the builder will publish again (its work queue " + PublishesLocal + ", design 47 G section 588, H2 section 793): " +
|
||||
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, pulled first, or again, asked again at a start) " +
|
||||
"with the oldest and newest queue time; the first 50 in the order they are pulled, each with its class, its artifacts' digests and when it was " +
|
||||
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, or again, asked again at a start) " +
|
||||
"with the oldest and newest queue time; the first 50 in stream order, the order each holder's one consumer pulls them, each with its class, its artifacts' digests and when it was " +
|
||||
"queued (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
|
||||
"sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " +
|
||||
"tries and next try, and why one ended (its tries spent, or dropped). A read past its pages is answered capped, naming the items read and the " +
|
||||
"queue's size (H-A-293). Read-only."
|
||||
|
||||
// PublishDropDescription and PublishFirstDescription are builder_publish_drop's and builder_publish_first's (§793).
|
||||
// PublishDropDescription is builder_publish_drop's (§793).
|
||||
const PublishDropDescription = "Drop one version's publish (design 47 H2 section 793): its items on the work queue " + PublishesLocal + " are " +
|
||||
"terminated at every holder, and its publish item ended with who dropped it and why, so it is never sent or put again; builder_publish_again " +
|
||||
"asks it anew. A hand act, recorded in the hand-act log first."
|
||||
const PublishFirstDescription = "Put one version's publish first (design 47 H2 section 793): each of its again items on the work queue " +
|
||||
PublishesLocal + " is put again as live, under the message id <holder>.live.<module>.<version>, pulled before every again item, and the old " +
|
||||
"terminated. A hand act, recorded in the hand-act log first."
|
||||
|
||||
// PublishAgainDescription is builder_publish_again's description.
|
||||
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -404,8 +405,9 @@ func answerOf(queue, acted, reasks Reader, module, version string) (Queued, erro
|
||||
}
|
||||
|
||||
// TestThePublishQueueToolAnswersAtAnySize: design 47 H H-A-290 (5) — builder_publish_queue over two thousand queued versions answers
|
||||
// within a second and small: counts per holder and class with the oldest and newest queue time, a list capped at fifty in the
|
||||
// order they are sent, and one module's item with its class; each state read in bulk where the reader can.
|
||||
// within a second and small: counts per holder and class with the oldest and newest queue time, a list capped at fifty in stream
|
||||
// order (H-A-483, third addendum: each key is its item's stream sequence), and one module's item with its class; each state read
|
||||
// in bulk where the reader can.
|
||||
func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
at := time.Date(2026, 11, 20, 17, 0, 0, 0, time.UTC)
|
||||
vals := map[string]json.RawMessage{}
|
||||
@@ -418,7 +420,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
class = PublishClassLive
|
||||
}
|
||||
q := QueuedPublish{V: 1, Holder: holder, Module: fmt.Sprintf("m%d", i), Version: "0.1.0", Artifacts: map[string]string{"a": "sha256:aa"}, Queued: at.Add(time.Duration(i) * time.Second), Class: class}
|
||||
vals[QueueKey(holder, q.Module, q.Version)], _ = json.Marshal(q)
|
||||
vals[strconv.Itoa(i+1)], _ = json.Marshal(q) // the reader keys each item by its stream sequence
|
||||
}
|
||||
gets := 0
|
||||
queue := Reader{All: func() (StatePage, error) { return StatePage{Values: vals}, nil },
|
||||
@@ -435,8 +437,10 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
if fmt.Sprintln(a.Counts[1].Holder, a.Counts[1].Class, a.Counts[1].Count, a.Counts[1].Oldest.Equal(at.Add(1991*time.Second)), a.Counts[1].Newest.Equal(at.Add(1999*time.Second))) != "go "+PublishClassLive+" 5 true true\n" || len(a.Counts) != 4 {
|
||||
t.Errorf("counts %+v, want per holder and class with their oldest and newest", a.Counts)
|
||||
}
|
||||
if a.Queue[0].Class != PublishClassLive || a.Queue[0].Holder != "go" {
|
||||
t.Errorf("the list opens with %+v, want the go holder's live class, sent first", a.Queue[0])
|
||||
for i, q := range a.Queue {
|
||||
if q.Module != fmt.Sprintf("m%d", i) {
|
||||
t.Fatalf("the list holds %s at %d, want m%d: the items in stream order, whatever their holder or class", q.Module, i, i)
|
||||
}
|
||||
}
|
||||
one, err := answerOf(queue, none, none, "m1995", "")
|
||||
if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 {
|
||||
|
||||
@@ -10,10 +10,8 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -42,8 +40,8 @@ const (
|
||||
// VersionHeldElsewhere is the finding of a number a registry seat already holds, skipped at reserving or moved from (design 47
|
||||
// H H-A-275), scope module.
|
||||
VersionHeldElsewhere = "version-held-elsewhere"
|
||||
// VersionTaggedElsewhere is the finding of a number whose git tag stands at another commit, skipped at reserving (design 47
|
||||
// H2 §796), scope module; cleared at the module's next version-released.
|
||||
// VersionTaggedElsewhere is the finding of a number whose git tag stood at another commit, which a reserve skipped before no
|
||||
// tag was read (design 47 H2 H-A-494); one raised before is cleared at the module's next version-released.
|
||||
VersionTaggedElsewhere = "version-tagged-elsewhere"
|
||||
)
|
||||
|
||||
@@ -55,6 +53,10 @@ type ReservedVersion struct {
|
||||
Version string `json:"version"` // MAJOR.MINOR.PATCH
|
||||
// Artifacts are the module's artifacts at the version: the digest once built, empty before.
|
||||
Artifacts map[string]string `json:"artifacts"`
|
||||
// Upstream are the artifacts of kind upstream, by reference (design 47 H2 H-A-486): kept in Artifacts with their digest for the
|
||||
// record, never awaited, as design 51 copies an upstream image by its pinned digest and no registry seat says it published at
|
||||
// a version. A reservation written before carries none.
|
||||
Upstream []string `json:"upstream,omitempty"`
|
||||
// BuiltAt is when the build ended with every digest known; the publish's bound counts from it.
|
||||
BuiltAt time.Time `json:"built-at,omitzero"`
|
||||
// Published are the artifacts a registry seat said published at this version with the built digest.
|
||||
@@ -63,8 +65,8 @@ type ReservedVersion struct {
|
||||
// was kept.
|
||||
PublishedAt map[string]time.Time `json:"published-at,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go
|
||||
// package's, GoTagOf): empty for a module with no Go package (design 47 G §370), or whose Go package's directory is not known.
|
||||
// Repository and Commit are what the build was made from. Tag is kept for a reservation written before design 47 H2 H-A-494:
|
||||
// no git tag is written at a release any more, and a new reservation names none.
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
@@ -110,22 +112,19 @@ type KeptPublished struct {
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
|
||||
// written at, and whether the module has a Go package, whose version is tagged.
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit, kept on the reservation.
|
||||
type Source struct {
|
||||
Repository, Commit string
|
||||
// Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76).
|
||||
Fingerprint string
|
||||
GoPackage bool
|
||||
// GoArtifact is the artifact that is the module's Go package, whose module path names the directory its tag is written in
|
||||
// (design 47 H2 §796).
|
||||
GoArtifact string
|
||||
// Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of
|
||||
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
|
||||
Branch string
|
||||
Revision int64
|
||||
// CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did.
|
||||
CatalogRevision int64
|
||||
// Upstream are the artifacts of kind upstream among those reserved, by reference (H-A-486), kept on the reservation.
|
||||
Upstream []string
|
||||
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
|
||||
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
|
||||
Rebuild bool
|
||||
@@ -171,92 +170,6 @@ type Reserver struct {
|
||||
HeldNumbers func(ctx context.Context, module, fingerprint string, refs []string) ([]string, error)
|
||||
// HeldElsewhere is told the numbers of module a reserve skipped because a registry holds them (H-A-275); nil is told none.
|
||||
HeldElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
// Names answers a Go package's module path, which names the directory its tag is written in (design 47 H2 §796); nil, or a
|
||||
// name not known, tags the version nowhere, said.
|
||||
Names PackageNames
|
||||
// Tags reads repository's tags starting with prefix, each with the commit it names, through the seat git's tags (design 47
|
||||
// H2 §796); away says the read met a transport fault or the error kind away, which lets the reservation go on, said. Nil
|
||||
// reads none.
|
||||
Tags func(ctx context.Context, repository, prefix string) (tags []git.Tag, away bool, err error)
|
||||
// TaggedElsewhere is told the numbers of module a reserve skipped because their tag stands at another commit (§796); nil
|
||||
// is told none.
|
||||
TaggedElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
|
||||
tagsMu sync.Mutex
|
||||
// tagsRead are the tags last read of each module, by series patch to the commit, with the catalog revision they were read
|
||||
// under: a module's tags are read once per catalog revision it reserves under (§796).
|
||||
tagsRead map[string]tagsAt
|
||||
}
|
||||
|
||||
// tagsAt are a module's tags of one series read under one catalog revision: patch to the commit its tag names.
|
||||
type tagsAt struct {
|
||||
revision, prefix string
|
||||
commits map[int]string
|
||||
}
|
||||
|
||||
// goDirectory is the directory of module's Go package in its repository (§796), from its module path; ok is false, said, when
|
||||
// no name or no directory of the repository is known.
|
||||
func (r *Reserver) goDirectory(ctx context.Context, module string, src Source) (string, bool) {
|
||||
if r.Names == nil {
|
||||
return "", false
|
||||
}
|
||||
path, known, err := r.Names(ctx, module, src.GoArtifact)
|
||||
switch {
|
||||
case err != nil:
|
||||
r.said(fmt.Errorf("%s: its Go package's name cannot be read, so its version is tagged nowhere: %w", module, err))
|
||||
return "", false
|
||||
case !known:
|
||||
r.said(fmt.Errorf("%s: no name of its Go package %s is known yet, so its version is tagged nowhere", module, src.GoArtifact))
|
||||
return "", false
|
||||
}
|
||||
dir, ok := GoDirectory(path, src.Repository)
|
||||
if !ok {
|
||||
r.said(fmt.Errorf("%s: its Go package %s is not a module path of the repository %s, so its version is tagged nowhere", module, path, src.Repository))
|
||||
}
|
||||
return dir, ok
|
||||
}
|
||||
|
||||
// taggedAt are the patches of series whose tag `<directory>/v<series>.<patch>` stands, each with its commit, read once per
|
||||
// catalog revision the module reserves under (§796). A read that meets a transport fault or away answers none, said, and the
|
||||
// reservation goes on; any other failure refuses the reservation, naming it.
|
||||
func (r *Reserver) taggedAt(ctx context.Context, module, series, dir string, src Source) (map[int]string, error) {
|
||||
if r.Tags == nil {
|
||||
return nil, nil
|
||||
}
|
||||
prefix := TagPrefix(dir) + series + "."
|
||||
revision := ""
|
||||
if src.CatalogRevision > 0 {
|
||||
revision = fmt.Sprintf("%s.%d", src.Branch, src.CatalogRevision)
|
||||
r.tagsMu.Lock()
|
||||
got, ok := r.tagsRead[module]
|
||||
r.tagsMu.Unlock()
|
||||
if ok && got.revision == revision && got.prefix == prefix {
|
||||
return got.commits, nil
|
||||
}
|
||||
}
|
||||
tags, away, err := r.Tags(ctx, src.Repository, prefix)
|
||||
if err != nil && away {
|
||||
r.said(fmt.Errorf("%s: its tags %s* cannot be read (%w), so the reservation goes on without them", module, prefix, err))
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the tags %s* of %s cannot be read: %w", prefix, module, err)
|
||||
}
|
||||
commits := map[int]string{}
|
||||
for _, t := range tags {
|
||||
if s, p, err := SplitVersion(strings.TrimPrefix(t.Tag, TagPrefix(dir))); err == nil && s == series && strings.HasPrefix(t.Tag, prefix) {
|
||||
commits[p] = t.Commit
|
||||
}
|
||||
}
|
||||
if revision != "" {
|
||||
r.tagsMu.Lock()
|
||||
if r.tagsRead == nil {
|
||||
r.tagsRead = map[string]tagsAt{}
|
||||
}
|
||||
r.tagsRead[module] = tagsAt{revision: revision, prefix: prefix, commits: commits}
|
||||
r.tagsMu.Unlock()
|
||||
}
|
||||
return commits, nil
|
||||
}
|
||||
|
||||
// heldNumbers are the patches of series a registry holds of module's artifacts (H-A-275); a read that fails is said, and none.
|
||||
@@ -277,19 +190,15 @@ func (r *Reserver) heldNumbers(ctx context.Context, module, series, fingerprint
|
||||
return out
|
||||
}
|
||||
|
||||
// nextFree is the first patch after after that no registry holds and whose tag stands at no other commit than commit, with the
|
||||
// held numbers it skipped (H-A-275) and those tagged elsewhere (design 47 H2 §796).
|
||||
func nextFree(series string, after int, held map[int]bool, tagged map[int]string, commit string) (int, []string, []string) {
|
||||
var skipped, elsewhere []string
|
||||
// nextFree is the first patch after after that no registry holds, with the held numbers it skipped (H-A-275). No git tag is
|
||||
// read: a series is numbered from the catalog's versions and the registries' alone (design 47 H2 H-A-494).
|
||||
func nextFree(series string, after int, held map[int]bool) (int, []string) {
|
||||
var skipped []string
|
||||
for p := after + 1; ; p++ {
|
||||
switch c, ok := tagged[p]; {
|
||||
case held[p]:
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
case ok && c != commit:
|
||||
elsewhere = append(elsewhere, fmt.Sprintf("%s.%d (tagged at %s)", series, p, c))
|
||||
default:
|
||||
return p, skipped, elsewhere
|
||||
if !held[p] {
|
||||
return p, skipped
|
||||
}
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,15 +251,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
}
|
||||
key := ReservedKey(module, series)
|
||||
held := r.heldNumbers(ctx, module, series, src.Fingerprint, artifacts)
|
||||
dir, tagged := "", false
|
||||
var tags map[int]string
|
||||
if src.GoPackage {
|
||||
if dir, tagged = r.goDirectory(ctx, module, src); tagged {
|
||||
if tags, err = r.taggedAt(ctx, module, series, dir, src); err != nil {
|
||||
return ReservedVersion{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
for {
|
||||
cur, rev, found, err := r.get(ctx, key)
|
||||
if err != nil {
|
||||
@@ -368,15 +268,13 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
next = p
|
||||
}
|
||||
}
|
||||
patch, skipped, elsewhere := nextFree(series, next, held, tags, src.Commit)
|
||||
patch, skipped := nextFree(series, next, held)
|
||||
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, patch), Artifacts: map[string]string{}, At: r.now(),
|
||||
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
|
||||
if tagged {
|
||||
out.Tag = GoTagOf(dir, out.Version)
|
||||
}
|
||||
for _, a := range artifacts {
|
||||
out.Artifacts[a] = ""
|
||||
}
|
||||
out.Upstream = slices.Clone(src.Upstream)
|
||||
if found {
|
||||
out.Withdrawn = maps.Clone(cur.Withdrawn)
|
||||
if cur.Fingerprint != src.Fingerprint {
|
||||
@@ -410,9 +308,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
if len(skipped) > 0 && r.HeldElsewhere != nil {
|
||||
r.HeldElsewhere(ctx, module, skipped)
|
||||
}
|
||||
if len(elsewhere) > 0 && r.TaggedElsewhere != nil {
|
||||
r.TaggedElsewhere(ctx, module, elsewhere)
|
||||
}
|
||||
if found && r.Took != nil {
|
||||
r.Took(ctx, cur, out)
|
||||
}
|
||||
@@ -1134,11 +1029,11 @@ const (
|
||||
ReservedVerb = meshbuilds.VerbReserved
|
||||
)
|
||||
|
||||
// Missing are the artifacts of v no registry seat said published, sorted.
|
||||
// Missing are the artifacts of v no registry seat said published, sorted; an upstream one is never awaited (H-A-486).
|
||||
func (v ReservedVersion) Missing() []string {
|
||||
var out []string
|
||||
for a := range v.Artifacts {
|
||||
if !v.Published[a] {
|
||||
if !v.Published[a] && !slices.Contains(v.Upstream, a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
)
|
||||
|
||||
@@ -171,6 +169,23 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
if got := cur.Missing(); !reflect.DeepEqual(got, []string{"m:b"}) {
|
||||
t.Errorf("missing = %v, want m:b", got)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved and recorded with its digest, but never awaited: once the others are published
|
||||
// nothing is missing.
|
||||
up, err := r.Reserve(ctx, "u", "0.1", []string{"u:server", "u:tools"}, Source{Upstream: []string{"u:server"}})
|
||||
if err != nil || !reflect.DeepEqual(up.Upstream, []string{"u:server"}) {
|
||||
t.Fatalf("reserved %+v (%v), want u:server kept as upstream", up, err)
|
||||
}
|
||||
for _, a := range []string{"u:server", "u:tools"} {
|
||||
if _, err := r.Built(ctx, "u", up.Version, a, "sha256:"+a, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if marked, _, err := r.Published(ctx, "u-tools", up.Version, "sha256:u:tools"); err != nil || len(marked) != 1 {
|
||||
t.Fatalf("the published of u:tools marked %v (%v)", marked, err)
|
||||
}
|
||||
if cur, _, _ := r.At(ctx, ReservedKey("u", "0.1")); len(cur.Missing()) != 0 || cur.Artifacts["u:server"] != "sha256:u:server" {
|
||||
t.Errorf("missing = %v with artifacts %v, want none missing and the upstream digest kept", cur.Missing(), cur.Artifacts)
|
||||
}
|
||||
if key, ok := ReservedBoundOf(ReservedBoundKey("m", "1.4")); !ok || key != ReservedKey("m", "1.4") || ReservedBoundFor(key) != ReservedBoundKey("m", "1.4") {
|
||||
t.Errorf("the bound key does not name the reservation back: %q %v", key, ok)
|
||||
}
|
||||
@@ -399,7 +414,7 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
return held, readErr
|
||||
},
|
||||
HeldElsewhere: func(_ context.Context, _ string, s []string) { skipped = append(skipped, s...) }}
|
||||
v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true})
|
||||
v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1"})
|
||||
if err != nil || v.Version != "0.1.39" || len(skipped) != 2 {
|
||||
t.Fatalf("reserved %q (%v), skipped %v: want 0.1.39 past the two numbers the registry holds", v.Version, err, skipped)
|
||||
}
|
||||
@@ -413,9 +428,9 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
}
|
||||
held = []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40", "0.2.0"}
|
||||
// The build again of a done key reserves as a rebuild (A-490b): a withdrawn number is never its floor's to reuse.
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true, Rebuild: true})
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1", Rebuild: true})
|
||||
if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != "" {
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (no Go package name known)", again, err)
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (design 47 H2 H-A-494)", again, err)
|
||||
}
|
||||
if at, _ := r.AtVersion(ctx, "0.1.39"); len(at) != 0 {
|
||||
t.Errorf("the index by version still names the withdrawn number: %v", at)
|
||||
@@ -428,50 +443,4 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
if v, err := r.Reserve(ctx, "node-tools", "0.2", []string{"node-tools:program"}, Source{Fingerprint: "fp1"}); err != nil || v.Version != "0.2.1" {
|
||||
t.Errorf("with the registry down reserved %q (%v), want 0.2.1 (what it answered still skipped)", v.Version, err)
|
||||
}
|
||||
// Design 47 H2 §796: a Go package's tags are read before it reserves, once per catalog revision; a number whose tag
|
||||
// `<directory>/v<version>` stands at another commit is skipped and said, one at the build's own commit is not; the version's
|
||||
// tag names the Go module's directory; a read met away lets the reservation go on, any other failure refuses it.
|
||||
r.HeldNumbers = nil
|
||||
var tagReads int
|
||||
var tagsErr error
|
||||
var tagsAway bool
|
||||
var elsewhere []string
|
||||
tags := []git.Tag{{Tag: "modules/nt/v0.3.0", Commit: "c-other"}, {Tag: "modules/nt/v0.3.1", Commit: "c-mine"}, {Tag: "modules/nt/v0.3.2", Commit: "c-other2"}}
|
||||
r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/nt", module == "node-tools" && artifact == "lib", nil
|
||||
}
|
||||
r.Tags = func(_ context.Context, repository, prefix string) ([]git.Tag, bool, error) {
|
||||
tagReads++
|
||||
if repository != "o/r" || prefix != "modules/nt/v0.3." {
|
||||
t.Errorf("tags read of %s %q, want o/r modules/nt/v0.3.", repository, prefix)
|
||||
}
|
||||
return tags, tagsAway, tagsErr
|
||||
}
|
||||
r.TaggedElsewhere = func(_ context.Context, _ string, s []string) { elsewhere = append(elsewhere, s...) }
|
||||
at := Source{Fingerprint: "fp3", GoPackage: true, GoArtifact: "lib", Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 7}
|
||||
v3, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at)
|
||||
if err != nil || v3.Version != "0.3.1" || v3.Tag != "modules/nt/v0.3.1" || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.0") {
|
||||
t.Fatalf("reserved %+v (%v), tagged elsewhere %v: want 0.3.1 (0.3.0 tagged at another commit, 0.3.1 at the build's own), tag modules/nt/v0.3.1", v3, err, elsewhere)
|
||||
}
|
||||
at.Fingerprint, at.Commit, elsewhere = "fp4", "c-new", nil
|
||||
if v4, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v4.Version != "0.3.3" || tagReads != 1 || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.2") {
|
||||
t.Fatalf("reserved %+v (%v) after %d reads, tagged elsewhere %v: want 0.3.3 past 0.3.2's tag, the tags read once for revision main.7", v4, err, tagReads, elsewhere)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsErr, tagsAway = 8, "fp5", errors.New("the forge is away"), true
|
||||
if v5, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v5.Version != "0.3.4" || tagReads != 2 {
|
||||
t.Fatalf("with the tags away reserved %+v (%v) after %d reads, want 0.3.4 and the reservation going on", v5, err, tagReads)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsAway = 9, "fp6", false
|
||||
if _, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err == nil || !strings.Contains(err.Error(), "tags") {
|
||||
t.Errorf("a tags read refused not away reserved anyway (%v), want the reservation refused naming the tags", err)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/novox/mesh-sdk/go", "https://git.example/novox/mesh-sdk.git"); !ok || d != "go" || GoTagOf(d, "0.1.9") != "go/v0.1.9" {
|
||||
t.Errorf("mesh-sdk's Go package: directory %q (%v), want go and the tag go/v0.1.9", d, ok)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/o/r", "o/r"); !ok || d != "" || GoTagOf(d, "1.0.0") != "v1.0.0" {
|
||||
t.Errorf("a root Go module: directory %q (%v), want the root and the tag v1.0.0", d, ok)
|
||||
}
|
||||
if _, ok := GoDirectory("git.example/o/rr/x", "o/r"); ok {
|
||||
t.Error("a module path of another repository was given a directory")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package versions
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// GoTagOf is the git tag a Go package's released version is written as (design 47 H2 §796): `<directory>/v<version>`, the
|
||||
// directory of its Go module in the repository, which the Go toolchain's direct fallback reads; `v<version>` at the root. The
|
||||
// `go/v<version>` tags go from new versions.
|
||||
func GoTagOf(directory, version string) string { return TagPrefix(directory) + version }
|
||||
|
||||
// GoDirectory is the directory of the Go module path modulePath in repository (`<owner>/<name>`, or a clone address naming
|
||||
// it): the part of the path after the repository's own, as the Go toolchain reads a module kept in a subdirectory; empty at
|
||||
// the root. ok is false when the path is not one of that repository.
|
||||
func GoDirectory(modulePath, repository string) (string, bool) {
|
||||
repo := "/" + strings.ToLower(trunk.RepositoryName(repository))
|
||||
lower := strings.ToLower(modulePath)
|
||||
for from := 0; ; {
|
||||
i := strings.Index(lower[from:], repo)
|
||||
if i < 0 || repo == "/" {
|
||||
return "", false
|
||||
}
|
||||
end := from + i + len(repo)
|
||||
if end == len(lower) || lower[end] == '/' {
|
||||
return strings.Trim(modulePath[end:], "/"), true
|
||||
}
|
||||
from = end
|
||||
}
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
// Design 51 *The floor*: the floor's three sources, the registry seats' versions, git.tags and indexer_requirements.
|
||||
// The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3).
|
||||
if want := []string{"seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
|
||||
"seat:git.tag", "seat:git.tags",
|
||||
"seat:git.tags", // the floor's source alone: no tag is written at a release (design 47 H2 H-A-494)
|
||||
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
|
||||
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions",
|
||||
// A-231: the build asks on each build seat's queue delivery (accept.delivery.<asker>).
|
||||
@@ -199,15 +199,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 {
|
||||
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
|
||||
}
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 ||
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 13 ||
|
||||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
|
||||
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
|
||||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime ||
|
||||
// H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default
|
||||
m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime ||
|
||||
m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
// H-A-492: the record of a version's digests runs under the builder's own bound, a setting of 60 s by default
|
||||
m.Config["record-bound-seconds"].Default != float64(60) ||
|
||||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 {
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), record-bound-seconds of 60 (H-A-492), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
}
|
||||
// Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends.
|
||||
if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
@@ -266,7 +268,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("health = %v, want kind unit alone", p["health"])
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
if len(env) != 9 || env["MESH_BUILDER_RECORD_BOUND_FILE"] != "${dir:state}/record-bound-seconds" || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
|
||||
}
|
||||
for k, v := range env {
|
||||
@@ -274,14 +276,16 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("env %s = %v: a setting reaches the process as a file, never its environment (A-71)", k, v)
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "file" && r["path"] == "${dir:state}/toolchain-wait-minutes" && r["content"] == "${config:toolchain-wait-minutes}" {
|
||||
found = true
|
||||
for _, setting := range []string{"toolchain-wait-minutes", "record-bound-seconds"} { // H-A-492: the record's bound too
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "file" && r["path"] == "${dir:state}/"+setting && r["content"] == "${config:"+setting+"}" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no file resource with the content ${config:%s}: A-71", setting)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no file resource with the content ${config:toolchain-wait-minutes}: A-71")
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "settings" {
|
||||
|
||||
@@ -91,7 +91,6 @@
|
||||
"invokes": [
|
||||
"seat:container-image-registry.remove",
|
||||
"seat:container-image-registry.versions",
|
||||
"seat:git.tag",
|
||||
"seat:git.tags",
|
||||
"seat:go-package-registry.remove",
|
||||
"seat:go-package-registry.versions",
|
||||
@@ -183,6 +182,14 @@
|
||||
"trusted": false,
|
||||
"default": 10
|
||||
},
|
||||
"record-bound-seconds": {
|
||||
"type": "number",
|
||||
"description": "seconds the builder gives the record of a version's digests after its build ends, under its own bound and never the build's (design 47 H2 H-A-492).",
|
||||
"minimum": 1,
|
||||
"maximum": 3600,
|
||||
"trusted": false,
|
||||
"default": 60
|
||||
},
|
||||
"ask-taken-bound": {
|
||||
"type": "string",
|
||||
"description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).",
|
||||
@@ -310,6 +317,14 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:package-wait-minutes}"
|
||||
},
|
||||
{
|
||||
"id": "record-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/record-bound-seconds",
|
||||
"owner": "mesh-delivery-builder",
|
||||
"mode": "0600",
|
||||
"content": "${config:record-bound-seconds}"
|
||||
},
|
||||
{
|
||||
"id": "publish-pause",
|
||||
"type": "file",
|
||||
@@ -361,6 +376,7 @@
|
||||
"MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes",
|
||||
"MESH_BUILDER_REVISION_WAIT_FILE": "${dir:state}/revision-wait-minutes",
|
||||
"MESH_BUILDER_PACKAGE_WAIT_FILE": "${dir:state}/package-wait-minutes",
|
||||
"MESH_BUILDER_RECORD_BOUND_FILE": "${dir:state}/record-bound-seconds",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_FILE": "${dir:state}/publish-pause",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE": "${dir:state}/publish-pause-max",
|
||||
"MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries",
|
||||
|
||||
@@ -57,7 +57,7 @@ func VerbDescription(verb string) string {
|
||||
case VerbDrop:
|
||||
return "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; H-A-401; ADR 0664 §4)."
|
||||
case VerbPublishQueue:
|
||||
return "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size."
|
||||
return "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in stream order, the order each holder's one consumer pulls them (H-A-483); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size."
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -338,7 +338,7 @@ const (
|
||||
type PublishQueueAnswerQueueItem struct {
|
||||
// each artifact reference, <module>:<artifact>, with the digest to publish
|
||||
Artifacts map[string]string `json:"artifacts,omitempty"`
|
||||
// live, a build a pipeline's step waits on, pulled first; again, asked again at a start
|
||||
// live, a build a pipeline's step waits on; again, asked again at a start
|
||||
Class string `json:"class"`
|
||||
// the registry seat's holder the item is sent to, a module
|
||||
Holder string `json:"holder"`
|
||||
@@ -368,7 +368,7 @@ type PublishQueueAnswer struct {
|
||||
ItemsRead int64 `json:"items-read,omitempty"`
|
||||
// how many of total the answer lists
|
||||
Listed int64 `json:"listed"`
|
||||
// the first 50 items listed, per holder in the order they are pulled
|
||||
// the first 50 items listed, in stream order
|
||||
Queue []PublishQueueAnswerQueueItem `json:"queue"`
|
||||
// with capped: the queue's size in bytes
|
||||
QueueBytes int64 `json:"queue-bytes,omitempty"`
|
||||
|
||||
@@ -14,13 +14,13 @@ const Seat = "mesh-pipeline-runner"
|
||||
|
||||
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
|
||||
// a reader of a value of another form says "form N, this reader knows M".
|
||||
const Form = 7
|
||||
const Form = 8
|
||||
|
||||
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
|
||||
// whose fields changed while its forms did not (design 47 H2 §780).
|
||||
const (
|
||||
generatedForms = "7"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
generatedForms = "8"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionAgainNValue=693580eed498 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -69,7 +69,7 @@ func VerbDescription(verb string) string {
|
||||
case VerbRun:
|
||||
return "The run on request of one or more members, each a module at a named commit, through their pipelines; delivers nothing, replacing the checker's try (ADR 0666 section 18). Says run-started."
|
||||
case VerbStart:
|
||||
return "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log."
|
||||
return "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log. With modules, a pipeline at the head keyed <branch>.<revision>.again.<n> carrying the named modules whose newest entry has no passed pipeline, at the take-in's tiers; a module whose newest entry passed is refused by name. Exactly one of module or modules (design 47 H2 H-A-479)."
|
||||
case VerbShow:
|
||||
return "One pipeline: naming a mover, its steps alone; with no mover, each mover's row (its module, outcome and failed step), never every step of every mover, or the steps of any mover the step and state filters hold; and its step log, with the stores' fill (design 47 H2 §789, H-A-376). Reads only."
|
||||
case VerbList:
|
||||
@@ -354,6 +354,44 @@ func DecodePipelinesByBranchRevisionUnfinishedModuleValue(raw []byte) (Pipelines
|
||||
return v, formRead(raw), nil
|
||||
}
|
||||
|
||||
// PipelinesByBranchRevisionAgainNKey is the key of one value of the state pipelines under the form "<branch>.<revision>.again.<n>".
|
||||
func PipelinesByBranchRevisionAgainNKey(branch, revision, n string) string {
|
||||
return branch + "." + revision + ".again." + n
|
||||
}
|
||||
|
||||
// ParsePipelinesByBranchRevisionAgainNKey reads the parts of a key of the form "<branch>.<revision>.again.<n>", in the order of the form; the last result is false for a key of another form.
|
||||
func ParsePipelinesByBranchRevisionAgainNKey(key string) (string, string, string, bool) {
|
||||
rest := key
|
||||
v0, after0, found0 := strings.Cut(rest, ".")
|
||||
if !found0 || v0 == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
rest = after0
|
||||
v2, after2, found2 := strings.Cut(rest, ".again.")
|
||||
if !found2 || v2 == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
rest = after2
|
||||
if rest == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
v4 := rest
|
||||
rest = ""
|
||||
if rest != "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
return v0, v2, v4, true
|
||||
}
|
||||
|
||||
// DecodePipelinesByBranchRevisionAgainNValue reads one value of the state pipelines under the form "<branch>.<revision>.again.<n>" as another part wrote it: a field PipelinesByBranchRevisionAgainNValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
|
||||
func DecodePipelinesByBranchRevisionAgainNValue(raw []byte) (PipelinesByBranchRevisionAgainNValue, int, error) {
|
||||
var v PipelinesByBranchRevisionAgainNValue
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return v, 0, err
|
||||
}
|
||||
return v, formRead(raw), nil
|
||||
}
|
||||
|
||||
// PipelinesByStepsPipelineMoverKey is the key of one value of the state pipelines under the form "steps.<pipeline>.<mover>".
|
||||
func PipelinesByStepsPipelineMoverKey(pipeline, mover string) string {
|
||||
return "steps." + pipeline + "." + mover
|
||||
@@ -1244,7 +1282,10 @@ type RunAnswer struct {
|
||||
// StartInput is what start takes.
|
||||
type StartInput struct {
|
||||
Branch string `json:"branch"`
|
||||
Module string `json:"module"`
|
||||
// the one mover of the head's pipeline <branch>.<revision>; absent when modules is given
|
||||
Module string `json:"module,omitempty"`
|
||||
// the modules started again at the head, each whose newest entry has no passed pipeline (a failed, cut or superseded one); absent when module is given (design 47 H2 H-A-479)
|
||||
Modules []string `json:"modules,omitempty"`
|
||||
// why, written with the caller the runtime proves in the hand-act log
|
||||
Why string `json:"why"`
|
||||
}
|
||||
@@ -1637,6 +1678,9 @@ type PipelinesByBranchRunRunValue = Pipeline
|
||||
// PipelinesByBranchRevisionUnfinishedModuleValue is one value of the state pipelines under the key <branch>.<revision>.unfinished.<module>.
|
||||
type PipelinesByBranchRevisionUnfinishedModuleValue = Pipeline
|
||||
|
||||
// PipelinesByBranchRevisionAgainNValue is one value of the state pipelines under the key <branch>.<revision>.again.<n>.
|
||||
type PipelinesByBranchRevisionAgainNValue = Pipeline
|
||||
|
||||
// PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps.<pipeline>.<mover>.
|
||||
type PipelinesByStepsPipelineMoverValue struct {
|
||||
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
|
||||
|
||||
@@ -156,7 +156,7 @@
|
||||
{
|
||||
"name": "publish-queue",
|
||||
"class": "read",
|
||||
"description": "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size.",
|
||||
"description": "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in stream order, the order each holder's one consumer pulls them (H-A-483); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -243,7 +243,7 @@
|
||||
"live",
|
||||
"again"
|
||||
],
|
||||
"description": "live, a build a pipeline's step waits on, pulled first; again, asked again at a start"
|
||||
"description": "live, a build a pipeline's step waits on; again, asked again at a start"
|
||||
},
|
||||
"artifacts": {
|
||||
"type": "object",
|
||||
@@ -266,7 +266,7 @@
|
||||
"queued"
|
||||
]
|
||||
},
|
||||
"description": "the first 50 items listed, per holder in the order they are pulled"
|
||||
"description": "the first 50 items listed, in stream order"
|
||||
},
|
||||
"listed": {
|
||||
"type": "integer",
|
||||
|
||||
@@ -50,12 +50,20 @@
|
||||
},
|
||||
{
|
||||
"name": "start",
|
||||
"description": "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log.",
|
||||
"description": "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log. With modules, a pipeline at the head keyed <branch>.<revision>.again.<n> carrying the named modules whose newest entry has no passed pipeline, at the take-in's tiers; a module whose newest entry passed is refused by name. Exactly one of module or modules (design 47 H2 H-A-479).",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"module": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "the one mover of the head's pipeline <branch>.<revision>; absent when modules is given"
|
||||
},
|
||||
"modules": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "the modules started again at the head, each whose newest entry has no passed pipeline (a failed, cut or superseded one); absent when module is given (design 47 H2 H-A-479)"
|
||||
},
|
||||
"branch": {
|
||||
"type": "string"
|
||||
@@ -66,7 +74,6 @@
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"module",
|
||||
"branch",
|
||||
"why"
|
||||
]
|
||||
@@ -910,6 +917,13 @@
|
||||
},
|
||||
"description": "a pipeline of its own of one mover kept in the runner's state unfinished, put in it by place-unfinished; the revision the unfinished entry names (design 47 H2 §86, H-A-398)"
|
||||
},
|
||||
{
|
||||
"key": "<branch>.<revision>.again.<n>",
|
||||
"value": {
|
||||
"$ref": "#/defs/Pipeline"
|
||||
},
|
||||
"description": "a pipeline started again at the head by start with modules, its movers the named modules whose newest entry has no passed pipeline; n from 1, taken by writing on absence (design 47 H2 H-A-479)"
|
||||
},
|
||||
{
|
||||
"key": "steps.<pipeline>.<mover>",
|
||||
"value": {
|
||||
@@ -2353,5 +2367,5 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"form": 7
|
||||
"form": 8
|
||||
}
|
||||
|
||||
@@ -251,7 +251,7 @@ export interface PublishQueueAnswerCountsItem {
|
||||
export interface PublishQueueAnswerQueueItem {
|
||||
/** each artifact reference, <module>:<artifact>, with the digest to publish */
|
||||
"artifacts"?: Record<string, string>;
|
||||
/** live, a build a pipeline's step waits on, pulled first; again, asked again at a start */
|
||||
/** live, a build a pipeline's step waits on; again, asked again at a start */
|
||||
"class": "live" | "again";
|
||||
/** the registry seat's holder the item is sent to, a module */
|
||||
"holder": string;
|
||||
@@ -275,7 +275,7 @@ export interface PublishQueueAnswer {
|
||||
"items-read"?: number;
|
||||
/** how many of total the answer lists */
|
||||
"listed": number;
|
||||
/** the first 50 items listed, per holder in the order they are pulled */
|
||||
/** the first 50 items listed, in stream order */
|
||||
"queue": PublishQueueAnswerQueueItem[];
|
||||
/** with capped: the queue's size in bytes */
|
||||
"queue-bytes"?: number;
|
||||
|
||||
@@ -121,6 +121,12 @@ export declare function pipelinesByBranchRevisionUnfinishedModuleKey(branch: str
|
||||
/** The parts of a key of the form "<branch>.<revision>.unfinished.<module>", or undefined for a key of another form. */
|
||||
export declare function parsePipelinesByBranchRevisionUnfinishedModuleKey(_key: string): { branch: string; revision: string; module: string } | undefined;
|
||||
|
||||
/** The key of one value of the state pipelines under the form "<branch>.<revision>.again.<n>". */
|
||||
export declare function pipelinesByBranchRevisionAgainNKey(branch: string, revision: string, n: string): string;
|
||||
|
||||
/** The parts of a key of the form "<branch>.<revision>.again.<n>", or undefined for a key of another form. */
|
||||
export declare function parsePipelinesByBranchRevisionAgainNKey(_key: string): { branch: string; revision: string; n: string } | undefined;
|
||||
|
||||
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
|
||||
export declare function pipelinesByStepsPipelineMoverKey(pipeline: string, mover: string): string;
|
||||
|
||||
@@ -699,7 +705,10 @@ export interface RunAnswer {
|
||||
/** is what start takes. */
|
||||
export interface StartInput {
|
||||
"branch": string;
|
||||
"module": string;
|
||||
/** the one mover of the head's pipeline <branch>.<revision>; absent when modules is given */
|
||||
"module"?: string;
|
||||
/** the modules started again at the head, each whose newest entry has no passed pipeline (a failed, cut or superseded one); absent when module is given (design 47 H2 H-A-479) */
|
||||
"modules"?: string[];
|
||||
/** why, written with the caller the runtime proves in the hand-act log */
|
||||
"why": string;
|
||||
}
|
||||
@@ -1031,6 +1040,9 @@ export type PipelinesByBranchRunRunValue = Pipeline;
|
||||
/** is one value of the state pipelines under the key <branch>.<revision>.unfinished.<module>. */
|
||||
export type PipelinesByBranchRevisionUnfinishedModuleValue = Pipeline;
|
||||
|
||||
/** is one value of the state pipelines under the key <branch>.<revision>.again.<n>. */
|
||||
export type PipelinesByBranchRevisionAgainNValue = Pipeline;
|
||||
|
||||
/** is one value of the state pipelines under the key steps.<pipeline>.<mover>. */
|
||||
export interface PipelinesByStepsPipelineMoverValue {
|
||||
/** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */
|
||||
|
||||
@@ -199,6 +199,29 @@ export function parsePipelinesByBranchRevisionUnfinishedModuleKey(_key) {
|
||||
return { branch, revision, module };
|
||||
}
|
||||
|
||||
/** The key of one value of the state pipelines under the form "<branch>.<revision>.again.<n>". */
|
||||
export function pipelinesByBranchRevisionAgainNKey(branch, revision, n) {
|
||||
return branch + "." + revision + ".again." + n;
|
||||
}
|
||||
|
||||
/** The parts of a key of the form "<branch>.<revision>.again.<n>", or undefined for a key of another form. */
|
||||
export function parsePipelinesByBranchRevisionAgainNKey(_key) {
|
||||
let _rest = _key;
|
||||
const _at0 = _rest.indexOf(".");
|
||||
if (_at0 <= 0) return undefined;
|
||||
const branch = _rest.slice(0, _at0);
|
||||
_rest = _rest.slice(_at0 + 1);
|
||||
const _at2 = _rest.indexOf(".again.");
|
||||
if (_at2 <= 0) return undefined;
|
||||
const revision = _rest.slice(0, _at2);
|
||||
_rest = _rest.slice(_at2 + 7);
|
||||
if (_rest === "") return undefined;
|
||||
const n = _rest;
|
||||
_rest = "";
|
||||
if (_rest !== "") return undefined;
|
||||
return { branch, revision, n };
|
||||
}
|
||||
|
||||
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
|
||||
export function pipelinesByStepsPipelineMoverKey(pipeline, mover) {
|
||||
return "steps." + pipeline + "." + mover;
|
||||
|
||||
@@ -105,9 +105,14 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
}
|
||||
}
|
||||
}
|
||||
// A pipeline that did not pass withdraws the reservations its revision holds (design 47 H2 §805, H-A-418 (1)).
|
||||
// A pipeline that did not pass withdraws the reservations its revision holds of its own movers, never another open
|
||||
// pipeline's at that revision (design 47 H2 §805, H-A-418 (1), H-A-479 second addendum).
|
||||
if ended.Outcome != runner.PipelineEndedOutcomePassed {
|
||||
if err := ix.Withdraw(ctx, ended.Branch, ended.Revision); err != nil {
|
||||
movers := make([]string, 0, len(ended.Movers))
|
||||
for _, m := range ended.Movers {
|
||||
movers = append(movers, m.Module)
|
||||
}
|
||||
if err := ix.Withdraw(ctx, ended.Branch, ended.Revision, movers); err != nil {
|
||||
return fmt.Errorf("the reservations of %s are not withdrawn: %w", BranchObject(ended.Branch, ended.Revision), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -100,9 +101,10 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said)
|
||||
}
|
||||
|
||||
// Design 47 H2 §805, H-A-418, H-A-420, H-A-424, H-A-436: a pipeline reserving twenty packages of mesh-sdk in main's
|
||||
// revision 2 keeps one re-pin revision of main when its gathering settles, at its first step-ended; its release then
|
||||
// keeps none on main, and a pipeline that did not pass withdraws its reservations back to the newest released version.
|
||||
// Design 47 H2 H-A-495 (ending H-A-418 (2), H-A-424 (4)), H-A-436: a pin names only a released version. A pipeline
|
||||
// reserving twenty packages of mesh-sdk in main's revision 2 moves no pin when its gathering settles; the release of
|
||||
// the reserved version re-pins g in one revision; a pipeline that did not pass withdraws its reservations and moves
|
||||
// nothing; and a pin held on a version never released is moved to the newest released at the next re-pin, said.
|
||||
ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) {
|
||||
out := map[string]map[string]string{}
|
||||
for _, d := range dirs {
|
||||
@@ -110,6 +112,8 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
var lines []string
|
||||
ix.Logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) }
|
||||
heard := newMem()
|
||||
ix.Heard = heard
|
||||
revision := BranchObject("main", 2)
|
||||
@@ -120,6 +124,7 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
}
|
||||
return rev.Revision
|
||||
}
|
||||
const sdkGo = "git.novox.be/novox/mesh-sdk/go"
|
||||
for i := range 20 {
|
||||
e := meshbuilds.VersionReservedEvent{Module: "mesh-sdk", Artifact: fmt.Sprintf("mesh-sdk:p%d", i), Path: "mesh-sdk/go", Version: "0.1.28", Revision: revision}
|
||||
if err := ix.HearVersionReserved(ctx, e); err != nil {
|
||||
@@ -129,35 +134,82 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
if _, _, open, _ := heard.Get(ctx, SettlingKey(revision)); !open || newest() != 2 {
|
||||
t.Fatalf("twenty reservations: gathering open %v, main's newest %d, want open and no revision kept before it settles", open, newest())
|
||||
}
|
||||
if err := ix.Settle(ctx, revision); err != nil || newest() != 3 {
|
||||
t.Fatalf("the gathering settled (%v) keeping main's newest %d, want one re-pin revision, 3", err, newest())
|
||||
if err := ix.Settle(ctx, revision); err != nil || newest() != 2 {
|
||||
t.Fatalf("the gathering settled (%v) keeping main's newest %d, want still 2: a reservation moves no pin", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" {
|
||||
t.Errorf("g after the reservations is pinned %v, want v0.1.28", now.Packages)
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.27" {
|
||||
t.Errorf("g after the reservations is pinned %v, want the released v0.1.27", now.Packages)
|
||||
}
|
||||
if err := ix.Settle(ctx, revision); err != nil || newest() != 3 {
|
||||
t.Errorf("a second step-ended settled (%v) keeping main's newest %d, want still 3", err, newest())
|
||||
}
|
||||
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.28", Commit: strings.Repeat("1", 40), Published: true}); err != nil {
|
||||
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: sdkGo, Version: "0.1.28", Commit: strings.Repeat("1", 40), Published: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ix.Repin(ctx, "mesh-sdk", "0.1.28"); err != nil || newest() != 3 {
|
||||
t.Errorf("the release of the reserved 0.1.28 re-pinned (%v) keeping main's newest %d, want no revision more", err, newest())
|
||||
t.Errorf("the release of the reserved 0.1.28 re-pinned (%v) keeping main's newest %d, want one revision more, 3", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.28" {
|
||||
t.Errorf("g after the release is pinned %v, want v0.1.28", now.Packages)
|
||||
}
|
||||
next := BranchObject("main", 3)
|
||||
if err := ix.HearVersionReserved(ctx, meshbuilds.VersionReservedEvent{Module: "mesh-sdk", Artifact: "mesh-sdk:p0", Path: "mesh-sdk/go", Version: "0.1.29", Revision: next}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Settle(ctx, next); err != nil || newest() != 4 {
|
||||
t.Fatalf("the reservation of 0.1.29 settled (%v) keeping main's newest %d, want 4", err, newest())
|
||||
if err := ix.Settle(ctx, next); err != nil || newest() != 3 {
|
||||
t.Fatalf("the reservation of 0.1.29 settled (%v) keeping main's newest %d, want still 3", err, newest())
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 3); err != nil || newest() != 5 {
|
||||
t.Fatalf("the withdrawal (%v) kept main's newest %d, want 5", err, newest())
|
||||
// H-A-479 second addendum: a pipeline at the same revision whose movers are others withdraws none of mesh-sdk's.
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"app"}); err != nil || newest() != 3 {
|
||||
t.Fatalf("another pipeline's withdrawal (%v) kept main's newest %d, want 3: nothing of mesh-sdk withdrawn", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" {
|
||||
if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); !kept {
|
||||
t.Fatalf("another pipeline's withdrawal took mesh-sdk's reservation")
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"mesh-sdk"}); err != nil || newest() != 3 {
|
||||
t.Fatalf("the withdrawal (%v) kept main's newest %d, want 3: the withdrawn reservation pinned nothing", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.28" {
|
||||
t.Errorf("g after the withdrawal is pinned %v, want the newest released v0.1.28", now.Packages)
|
||||
}
|
||||
if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); kept {
|
||||
t.Errorf("the withdrawn reservation is still kept")
|
||||
}
|
||||
stale, _ := moduleOf(t, mods, "g")
|
||||
stale.Packages = map[string]string{sdkGo: "v0.1.29"} // pinned before H-A-495 to the withdrawn reservation
|
||||
mods.put("g", mustEncode(t, stale))
|
||||
lines = nil
|
||||
if _, err := ix.Repin(ctx, "mesh-sdk", "0.1.28"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.28" {
|
||||
t.Errorf("g pinned to the never released v0.1.29 is pinned %v after a re-pin, want v0.1.28", now.Packages)
|
||||
}
|
||||
if said := "g: pinned to " + sdkGo + " v0.1.29, never released; moved to v0.1.28"; !slices.Contains(lines, said) {
|
||||
t.Errorf("the re-pin said %q, want the line %q", lines, said)
|
||||
}
|
||||
// H-A-479 second addendum: two pipelines' movers reserve at one revision while its gathering is open; withdrawing one
|
||||
// pipeline's mover keeps the gathering and the other's reservation, withdrawing the last removes the gathering too.
|
||||
six := BranchObject("main", 6)
|
||||
for _, m := range []string{"one", "two"} {
|
||||
raw, _ := json.Marshal(Reserved{V: 1, Revision: six, Module: m})
|
||||
if err := heard.Create(ctx, ReservedKey(six, m), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := heard.Create(ctx, SettlingKey(six), []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 6, []string{"one"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, gathering, _ := heard.Get(ctx, SettlingKey(six))
|
||||
_, _, one, _ := heard.Get(ctx, ReservedKey(six, "one"))
|
||||
_, _, two, _ := heard.Get(ctx, ReservedKey(six, "two"))
|
||||
if !gathering || one || !two {
|
||||
t.Fatalf("withdrawing one: gathering %v, one kept %v, two kept %v; want the gathering and two kept, one gone", gathering, one, two)
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 6, []string{"two"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, gathering, _ := heard.Get(ctx, SettlingKey(six)); gathering {
|
||||
t.Error("withdrawing the last reservation kept the revision's gathering")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,13 +17,13 @@ import (
|
||||
|
||||
// A release re-pins once, at its reservation (design 47 H2 §805; H-A-418, H-A-420, H-A-424). The builder says
|
||||
// version-reserved when it reserves a package's number, naming the revision `<branch>.<n>` its pipeline runs. The indexer
|
||||
// writes the series' numbered version with the contents read at the commit of the module's entry in that revision, so
|
||||
// acceptedSeries and releasedPins pin it, and keeps the reservation in its state heard under
|
||||
// writes the series' numbered version with the contents read at the commit of the module's entry in that revision, and
|
||||
// keeps the reservation in its state heard under
|
||||
// `reserved.<revision>.<module>`, each key living reserved-keep. It gathers a pipeline's reservations per revision and
|
||||
// writes one re-pin revision of that revision's branch when the gathering settles: at the first pipeline-started or
|
||||
// step-ended of that pipeline heard after them, or at the expiry of the key settling.<revision>, written reserved-settle
|
||||
// after the first. The release's package-published then
|
||||
// moves no fingerprint. A pipeline-ended not passed whose revision holds reservations withdraws them: the series'
|
||||
// after the first. A pin names only a released version, never a reservation (design 47 H2 H-A-495), so that re-pin
|
||||
// moves no pin, and the release's re-pin (H-A-165) moves the importers. A pipeline-ended not passed whose revision holds reservations withdraws them: the series'
|
||||
// numbered goes back to its newest released version and the importers are re-pinned to it.
|
||||
|
||||
// ReservedPrefix begins every key of the state heard holding a revision's reservations of one module.
|
||||
@@ -350,20 +350,35 @@ func (ix *Indexer) settleNow(ctx context.Context, revision string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Withdraw takes a pipeline-ended not passed of branch's revision in: each reservation the revision holds is withdrawn, its
|
||||
// series' numbered set back to the newest released version, its importers re-pinned to that, and its key deleted.
|
||||
func (ix *Indexer) Withdraw(ctx context.Context, branch string, revision int64) error {
|
||||
// Withdraw takes a pipeline-ended not passed of branch's revision in: each reservation the revision holds of one of the
|
||||
// ended pipeline's own movers is withdrawn, its series' numbered set back to the newest released version, its importers
|
||||
// re-pinned to that, and its key deleted. Another open pipeline's reservations at the same revision are kept, and the
|
||||
// revision's gathering is withdrawn only when none of them is left (design 47 H2 H-A-479 second addendum).
|
||||
func (ix *Indexer) Withdraw(ctx context.Context, branch string, revision int64, movers []string) error {
|
||||
if ix.Heard == nil {
|
||||
return nil
|
||||
}
|
||||
name := BranchObject(branch, revision)
|
||||
if _, rev, found, err := ix.Heard.Get(ctx, SettlingKey(name)); err == nil && found {
|
||||
_ = ix.Heard.Delete(ctx, SettlingKey(name), rev) // the gathering is withdrawn with it
|
||||
}
|
||||
held, revs, err := ix.reservedOf(ctx, name)
|
||||
if err != nil || len(held) == 0 {
|
||||
all, revs, err := ix.reservedOf(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held := map[string]Reserved{}
|
||||
for key, r := range all {
|
||||
if slices.Contains(movers, r.Module) {
|
||||
held[key] = r
|
||||
} else {
|
||||
delete(revs, key)
|
||||
}
|
||||
}
|
||||
if len(held) == len(all) {
|
||||
if _, rev, found, err := ix.Heard.Get(ctx, SettlingKey(name)); err == nil && found {
|
||||
_ = ix.Heard.Delete(ctx, SettlingKey(name), rev) // the gathering is withdrawn with it
|
||||
}
|
||||
}
|
||||
if len(held) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, r := range held {
|
||||
for _, one := range r.Reservations {
|
||||
if err := ix.withdrawNumbered(ctx, one); err != nil {
|
||||
|
||||
@@ -198,14 +198,10 @@ func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repos
|
||||
return out, pins
|
||||
}
|
||||
|
||||
// pinnedVersion is the version of a series its users are pinned to: its numbered one when a reservation wrote it with its
|
||||
// contents and it is newer than the newest released (design 47 H2 §805, H-A-418 (2), H-A-424 (4)), else the newest; nil
|
||||
// for none.
|
||||
// pinnedVersion is the version of a series its users are pinned to: its newest released one, never a reservation's
|
||||
// numbered one, which a pipeline that does not pass withdraws unpublished (design 47 H2 H-A-495, which ends H-A-418 (2)
|
||||
// and H-A-424 (4)); nil for none.
|
||||
func pinnedVersion(s SeriesPackage) *SeriesVersion {
|
||||
if n := s.Numbered; n != nil && n.Contents != nil && (s.Newest == nil || CompareVersions("v"+s.Newest.Version, "v"+n.Version) < 0) {
|
||||
p := publishedOfNumbered(*n.Contents)
|
||||
return &SeriesVersion{Version: n.Version, Commit: n.Commit, Digest: n.Digest, Fingerprint: n.Fingerprint, Contents: contentsOfPublished(p)}
|
||||
}
|
||||
return s.Newest
|
||||
}
|
||||
|
||||
|
||||
@@ -236,6 +236,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
fp, none string
|
||||
tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint
|
||||
pkgs sdkstep.PackageVersions // the package versions, when there is a fingerprint
|
||||
moved []string // the pins moved off a version never released, said (H-A-495)
|
||||
}
|
||||
done := map[string]result{}
|
||||
visiting := map[string]bool{}
|
||||
@@ -313,6 +314,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
// a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
|
||||
r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository, kept.Packages)
|
||||
r.tcs = toolchainsUsed(toolchains)
|
||||
r.moved = neverReleased(module, k.held.Packages, r.pkgs, series, k.manifest)
|
||||
}
|
||||
}
|
||||
if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
|
||||
@@ -325,6 +327,9 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
}
|
||||
done[module] = r
|
||||
k.cur.Inputs2 = &kept
|
||||
for _, line := range r.moved {
|
||||
ix.logf("%s", line)
|
||||
}
|
||||
return r
|
||||
}
|
||||
finals := map[string]meshcatalog.ModulesByModuleValue{}
|
||||
|
||||
@@ -140,7 +140,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
|
||||
}
|
||||
var want []string
|
||||
for _, a := range entry.Artifacts {
|
||||
want = append(want, a.Ref)
|
||||
if a.Kind != catalogue.ArtifactUpstream { // an upstream copy is released by its pinned digest, never published (H-A-486)
|
||||
want = append(want, a.Ref)
|
||||
}
|
||||
}
|
||||
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
|
||||
var have published
|
||||
@@ -211,7 +213,15 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
|
||||
at := ix.now().UTC()
|
||||
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
|
||||
pins, err := upstreamPins(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if a.Kind == catalogue.ArtifactUpstream {
|
||||
v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: a.Ref, Digest: pins[a.Ref]})
|
||||
continue
|
||||
}
|
||||
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
|
||||
}
|
||||
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
|
||||
@@ -255,6 +265,46 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
return ix.forgetPublished(ctx, pendingKey)
|
||||
}
|
||||
|
||||
// upstreamPins are the pinned digests of the entry's upstream artifacts, by reference, read from its manifest's from
|
||||
// (`<image>@sha256:<digest>`): design 51 copies an upstream image by that digest, so a release names it (H-A-486). An upstream
|
||||
// artifact whose manifest names no pin refuses the release, naming it.
|
||||
func upstreamPins(entry meshcatalog.CatalogEntry) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
if !slices.ContainsFunc(entry.Artifacts, func(a meshcatalog.Artifact) bool { return a.Kind == catalogue.ArtifactUpstream }) {
|
||||
return out, nil
|
||||
}
|
||||
raw, err := json.Marshal(entry.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
From string `json:"from"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s's entry does not read, so its upstream artifacts' pins are not known: %w", entry.Module, err)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactUpstream {
|
||||
continue
|
||||
}
|
||||
if _, d, ok := strings.Cut(a.From, "@"); ok && strings.HasPrefix(d, "sha256:") {
|
||||
out[ArtifactRefOf(entry.Module, a.Name)] = d
|
||||
}
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if _, ok := out[a.Ref]; a.Kind == catalogue.ArtifactUpstream && !ok {
|
||||
return nil, fmt.Errorf("%s is an upstream artifact whose manifest names no pinned digest (from <image>@sha256:<digest>), so %s releases no version", a.Ref, entry.Module)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
|
||||
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
|
||||
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
|
||||
@@ -423,6 +473,29 @@ func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, rep
|
||||
return fingerprint.WithPins(f.fp, pins), pins
|
||||
}
|
||||
|
||||
// neverReleased are the lines said of a module whose held pins (was) name a version above its series' newest released one,
|
||||
// a reservation withdrawn unpublished, and whose new pins move it (design 47 H2 H-A-495): "<module>: pinned to <package>
|
||||
// <version>, never released; moved to <version>". nil for none.
|
||||
func neverReleased(module string, was map[string]string, pins sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, p := range slices.Sorted(maps.Keys(was)) {
|
||||
old, now := strings.TrimPrefix(was[p], "v"), pins[p]
|
||||
if _, ok := versionParts(old); !ok || was[p] == now {
|
||||
continue
|
||||
}
|
||||
for _, s := range series {
|
||||
if s.GoModule != p || s.Newest == nil || s.Series != m.AcceptedSeries(s.Name) {
|
||||
continue
|
||||
}
|
||||
if in, _ := SeriesOf(old); in == s.Series && compareVersions(old, strings.TrimPrefix(s.Newest.Version, "v")) > 0 {
|
||||
out = append(out, fmt.Sprintf("%s: pinned to %s %s, never released; moved to %s", module, p, was[p], cmp.Or(now, "no version")))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moduleOfKey is the module a modules key names; "" for none.
|
||||
func moduleOfKey(key string) string {
|
||||
if key == "" {
|
||||
|
||||
@@ -83,6 +83,36 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
|
||||
if compareVersions(oldest, "0.1.5") < 0 {
|
||||
t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest)
|
||||
}
|
||||
|
||||
// H-A-486: an upstream artifact is copied by its pinned digest and never published, so the version is released once every
|
||||
// other artifact is, the upstream one named with the pin its manifest gives; one whose manifest names no pin releases none.
|
||||
pin := "sha256:" + strings.Repeat("ab", 32)
|
||||
nats := func(from string) map[string]meshcatalog.CatalogEntry {
|
||||
return map[string]meshcatalog.CatalogEntry{"nats": {Module: "nats", Manifest: map[string]any{"build": map[string]any{"artifacts": []any{
|
||||
map[string]any{"name": "server", "kind": "upstream", "from": from},
|
||||
map[string]any{"name": "tools", "kind": "bundle", "from": "."}}}},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "nats:server", Kind: "upstream"}, {Ref: "nats:tools", Kind: "bundle", Language: "go"}}}}
|
||||
}
|
||||
for i, c := range []struct {
|
||||
from, version string
|
||||
released bool
|
||||
}{{"nats:2.12-alpine@" + pin, "0.1.1", true}, {"nats:2.12-alpine", "0.1.2", false}} {
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: int64(4 + i), Kind: "trunk"}, nats(c.from))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "nats", Artifact: "nats:tools", Digest: "sha256:tools", Version: c.version, Source: 9, Branch: "main"})
|
||||
got, _ := ModuleVersions(ctx, ix.Versions, "nats")
|
||||
switch {
|
||||
case c.released && (err != nil || len(got) != 1 || len(got[0].Artifacts) != 2 || got[0].Artifacts[0] != (meshcatalog.VersionArtifact{Artifact: "nats:server", Digest: pin})):
|
||||
t.Fatalf("pinned %q: released %+v (%v); want the server named with its pin", c.from, got, err)
|
||||
case !c.released && (err == nil || !strings.Contains(err.Error(), "no pinned digest") || len(got) != 1):
|
||||
t.Fatalf("unpinned %q: released %+v (%v); want a refusal naming the pin", c.from, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or
|
||||
|
||||
@@ -27,9 +27,10 @@ func TestTheJudge(t *testing.T) {
|
||||
want: map[string]string{"a": `secret-`}},
|
||||
{name: "an unknown field", raws: map[string]string{"a": `{"module":"a","version":"1.0","a-field-from-later":1}`},
|
||||
want: map[string]string{"a": `"a-field-from-later"`}},
|
||||
// H-A-465: the take-in sets a top-level word it does not know aside (ForInputs); the judgement still refuses it.
|
||||
{name: "a word a newer SDK adds", raws: map[string]string{"a": `{"module":"a","version":"1.0","class":"core"}`},
|
||||
want: map[string]string{"a": `"class"`}},
|
||||
// H-A-465: the take-in sets a top-level word it does not know aside (ForInputs); the judgement still refuses it. The
|
||||
// word is one no SDK knows: class, the word of H-A-465's night, is known since H2.
|
||||
{name: "a word a newer SDK adds", raws: map[string]string{"a": `{"module":"a","version":"1.0","zzz-unknown-word":"core"}`},
|
||||
want: map[string]string{"a": `"zzz-unknown-word"`}},
|
||||
// Design 47 G §23, §28: a step's status is kept in the registration and never in a manifest.
|
||||
{name: "a manifest naming a step's status", raws: map[string]string{"a": `{"module":"a","version":"1.0","policy":{"lint":"bypass"}}`},
|
||||
want: map[string]string{"a": `"policy"`}},
|
||||
|
||||
@@ -19,11 +19,11 @@ func VerbTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.
|
||||
}
|
||||
return []stdio.Tool{
|
||||
verbTool[seat.StartInput](ask, "start", seat.VerbStart,
|
||||
"Start by hand the delivering pipeline of the head of a watched branch for a module, by the seat verb start: the branch, the module and why. A head whose pipeline is written already is refused by the verb, naming it.",
|
||||
obj([]string{"branch", "module", "why"}, map[string]any{"branch": str, "module": str, "why": str}),
|
||||
"Start by hand the delivering pipeline of the head of a watched branch for a module, by the seat verb start: the branch, the module and why. A head whose pipeline is written already is refused by the verb, naming it. With modules instead of module, the named modules whose newest entry has no passed pipeline (a failed, cut or superseded one) are started again at the head, keyed <branch>.<revision>.again.<n>; a module whose newest entry passed is refused by name (design 47 H2 H-A-479).",
|
||||
obj([]string{"branch", "why"}, map[string]any{"branch": str, "module": str, "modules": map[string]any{"type": "array", "items": str}, "why": str}),
|
||||
func(in seat.StartInput) string {
|
||||
if in.Branch == "" || in.Module == "" {
|
||||
return "start names the watched branch and the module"
|
||||
if in.Branch == "" || (in.Module == "") == (len(in.Modules) == 0) {
|
||||
return "start names the watched branch and exactly one of the module or the modules"
|
||||
}
|
||||
return whyOf(in.Why)
|
||||
}),
|
||||
|
||||
@@ -18,7 +18,11 @@ func TestVerbTools(t *testing.T) {
|
||||
{"start", "runner_start", "seat:mesh-pipeline-runner.start", "", map[string]any{"branch": "main", "module": "i3", "why": "cancelled"},
|
||||
seat.StartInput{Branch: "main", Module: "i3", Why: "cancelled"}},
|
||||
{"start no why", "runner_start", "", "names why", map[string]any{"branch": "main", "module": "i3"}, nil},
|
||||
{"start no module", "runner_start", "", "branch and the module", map[string]any{"branch": "main", "why": "x"}, nil},
|
||||
{"start no module", "runner_start", "", "exactly one of the module or the modules", map[string]any{"branch": "main", "why": "x"}, nil},
|
||||
// H-A-479: modules start the named modules again at the head; both together are refused.
|
||||
{"start modules", "runner_start", "seat:mesh-pipeline-runner.start", "", map[string]any{"branch": "main", "modules": []any{"i3", "lib"}, "why": "failed"},
|
||||
seat.StartInput{Branch: "main", Modules: []string{"i3", "lib"}, Why: "failed"}},
|
||||
{"start module and modules", "runner_start", "", "exactly one of the module or the modules", map[string]any{"branch": "main", "module": "i3", "modules": []any{"lib"}, "why": "x"}, nil},
|
||||
{"run", "runner_run", "seat:mesh-pipeline-runner.run", "", map[string]any{"members": []any{map[string]any{}}, "why": "x"},
|
||||
seat.RunInput{Members: []seat.Member{{}}, Why: "x"}},
|
||||
{"run no members", "runner_run", "", "members", map[string]any{}, nil},
|
||||
|
||||
@@ -96,3 +96,34 @@ func TestARevisionHeardIsHandedToItsStart(t *testing.T) {
|
||||
t.Errorf("a body that is no event = %v, want it ended", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-479 addendum (2): a revision is held when the catalog seat's revision verb answers that revision, read fresh; none-yet,
|
||||
// another revision or a refusal saying "no such object" is gone; another failure is said as it is.
|
||||
func TestRevisionHeldReadsTheCatalogsRevisionFresh(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
answer catalog.RevisionAnswer
|
||||
err error
|
||||
gone bool
|
||||
fails bool
|
||||
}{
|
||||
{name: "held", answer: catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 258}}},
|
||||
{name: "another revision", answer: catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 259}}, gone: true, fails: true},
|
||||
{name: "none yet", answer: catalog.RevisionAnswer{NoneYet: true}, gone: true, fails: true},
|
||||
{name: "removed", err: errors.New("revision main.258: no such object"), gone: true, fails: true},
|
||||
{name: "unreachable", err: errors.New("no responders"), fails: true},
|
||||
} {
|
||||
var asked catalog.RevisionInput
|
||||
call := func(_ string, body any) (json.RawMessage, error) {
|
||||
asked = body.(catalog.RevisionInput)
|
||||
if c.err != nil {
|
||||
return nil, c.err
|
||||
}
|
||||
return json.Marshal(c.answer)
|
||||
}
|
||||
err := revisionHeld(call)("main", 258)
|
||||
if (err != nil) != c.fails || errors.Is(err, pipeline.ErrRevisionGone) != c.gone || asked.Revision != 258 || asked.Branch != "main" {
|
||||
t.Errorf("%s: %v (asked %+v)", c.name, err, asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,10 +4,14 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
|
||||
@@ -65,5 +69,37 @@ func writeVerbs(h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipe
|
||||
candidate := func(group, branch string, revision int64) (pipeline.Half, error) {
|
||||
return readCandidate(calls.CallKept, catalog.CatalogRevisionEvent{Kind: "candidate", Group: group, Branch: branch, Revision: revision})
|
||||
}
|
||||
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}}
|
||||
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call},
|
||||
RevisionHeld: revisionHeld(calls.CallKept)}
|
||||
}
|
||||
|
||||
// revisionHeld reads the first page of a branch's catalog revision through the catalog seat's verb revision, never a kept
|
||||
// copy, so a rerun reopening an ended pipeline knows whether the catalog still keeps its revision (design 47 H2 H-A-479
|
||||
// addendum (2)): an answer of that revision is kept; none-yet, another revision, or a refusal saying the catalog's
|
||||
// "no such object" is gone (pipeline.ErrRevisionGone); any other failure is said as it is.
|
||||
func revisionHeld(call func(key string, body any) (json.RawMessage, error)) func(string, int64) error {
|
||||
return func(branch string, revision int64) error {
|
||||
raw, err := call(meshcatalogclient.RevisionVerb, catalog.RevisionInput{Branch: keys.EscapeName(branch), Revision: revision})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "no such object") {
|
||||
return fmt.Errorf("%w: %v", pipeline.ErrRevisionGone, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
answer, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
if strings.Contains(said, "no such object") {
|
||||
return fmt.Errorf("%w: %s", pipeline.ErrRevisionGone, said)
|
||||
}
|
||||
return fmt.Errorf("%s answered: %s", meshcatalogclient.RevisionVerb, said)
|
||||
}
|
||||
var a catalog.RevisionAnswer
|
||||
if err := json.Unmarshal(answer, &a); err != nil {
|
||||
return fmt.Errorf("%s answered no revision: %w", meshcatalogclient.RevisionVerb, err)
|
||||
}
|
||||
if a.NoneYet || a.Revision == nil || a.Revision.Revision != revision {
|
||||
return fmt.Errorf("%w: %s answered no revision %d of %s", pipeline.ErrRevisionGone, meshcatalogclient.RevisionVerb, revision, branch)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,8 +38,9 @@ func widenerOf(call func(key string, body any) (json.RawMessage, error), bound t
|
||||
const (
|
||||
envWidensReadBound = "MESH_RUNNER_WIDENS_READ_BOUND_FILE"
|
||||
envRevisionsKept = "MESH_RUNNER_REVISIONS_KEPT_FILE"
|
||||
// defaultReadBound is how long one read of widens waits for its answer: a few seconds, never the call's 30.
|
||||
defaultReadBound = 5 * time.Second
|
||||
// defaultReadBound is how long one read of widens waits for its answer: 30 s, so a cold grants-for (one that composes a
|
||||
// revision's grants the grants part did not keep yet) is not cut (design 47 H2 H-A-478).
|
||||
defaultReadBound = 30 * time.Second
|
||||
// defaultRevisionsKept is how many catalog revisions the runner keeps once read; a revision of the catalog graph is some 1.4 MB.
|
||||
defaultRevisionsKept = 6
|
||||
)
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestWidensSettings(t *testing.T) {
|
||||
kept int
|
||||
ok bool
|
||||
}{
|
||||
{"defaults", nil, 5 * time.Second, 6, true},
|
||||
{"defaults", nil, 30 * time.Second, 6, true},
|
||||
{"set", map[string]string{envWidensReadBound: "2s\n", envRevisionsKept: "3\n"}, 2 * time.Second, 3, true},
|
||||
{"no duration", map[string]string{envWidensReadBound: "5"}, 0, 0, false},
|
||||
{"no number", map[string]string{envRevisionsKept: "0"}, 0, 0, false},
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sync"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
@@ -28,23 +29,20 @@ func StepsKey(pipeline, mover string) string {
|
||||
// stepsPrefix is the start of every mover key of one pipeline.
|
||||
func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") }
|
||||
|
||||
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a
|
||||
// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`), an unfinished mover's own (6 parts, its fourth
|
||||
// `unfinished`, H-A-398) or a publish's (7 parts) and one mover.
|
||||
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.`, a pipeline's
|
||||
// key of any form the seat defines (a take-in's, a run's, a publish's, an unfinished mover's own or one started again at the head,
|
||||
// H-A-398, H-A-479), read by the one parse of the seat's key forms (design 47 H2 H-A-485), and one mover.
|
||||
func IsStepsKey(key string) bool {
|
||||
parts := strings.Split(key, ".")
|
||||
if parts[0] != "steps" {
|
||||
rest, ok := strings.CutPrefix(key, "steps.")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
switch len(parts) {
|
||||
case 4, 7:
|
||||
return true
|
||||
case 5:
|
||||
return parts[2] == runToken
|
||||
case 6:
|
||||
return parts[3] == unfinishedToken
|
||||
i := strings.LastIndex(rest, ".")
|
||||
if i <= 0 || i == len(rest)-1 {
|
||||
return false
|
||||
}
|
||||
return false
|
||||
_, err := pipelinekey.Parse(rest[:i])
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by
|
||||
|
||||
@@ -25,7 +25,7 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int,
|
||||
// A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key
|
||||
// form a source's pipelines were kept under.
|
||||
var pkeys []string
|
||||
for _, f := range []string{b + ".*", b + ".*.published.*.*", b + ".*." + unfinishedToken + ".*"} {
|
||||
for _, f := range []string{b + ".*", b + ".*.published.*.*", b + ".*." + unfinishedToken + ".*", b + ".*." + againToken + ".*"} {
|
||||
k, err := h.Store.Keys(f)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err)
|
||||
|
||||
@@ -69,8 +69,19 @@ func keyRevision(b Begin) int64 {
|
||||
}
|
||||
|
||||
// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, a publish's kept from before
|
||||
// a release revision started one pipeline (H-A-213 (2)), and an unfinished mover's pipeline of its own (H-A-398).
|
||||
var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*", "*.*." + unfinishedToken + ".*"}
|
||||
// a release revision started one pipeline (H-A-213 (2)), an unfinished mover's pipeline of its own (H-A-398), and a pipeline
|
||||
// started again at the head (H-A-479).
|
||||
var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*", "*.*." + unfinishedToken + ".*", "*.*." + againToken + ".*"}
|
||||
|
||||
// againToken is the third part of the key of a pipeline started again at the head by start with modules,
|
||||
// `<branch>.<revision>.again.<n>` (design 47 H2 H-A-479 addendum (1)).
|
||||
const againToken = "again"
|
||||
|
||||
// AgainPipelineKey is the key of the n-th pipeline started again at revision of branch, `<branch>.<revision>.again.<n>`
|
||||
// (H-A-479 addendum (1)).
|
||||
func AgainPipelineKey(branch string, revision int64, n int) string {
|
||||
return seat.PipelinesByBranchRevisionAgainNKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10), strconv.Itoa(n))
|
||||
}
|
||||
|
||||
// AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83).
|
||||
func AllKeys(store interface {
|
||||
|
||||
@@ -655,6 +655,50 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
|
||||
t.Error("the step's finding was written before its run's end")
|
||||
}
|
||||
})
|
||||
// H-A-479 addendum (2): a test step's rerun reopening an ended pipeline says its pipeline-started before its run is
|
||||
// started again, so its revision is pinned before the step is asked.
|
||||
t.Run("a test step's rerun reopening an ended pipeline pins its revision first", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, rev, _ := f.store.Get(p.Pipeline)
|
||||
q.Outcome, q.Ended = seat.PipelineOutcomeFailed, &ended
|
||||
if _, err := f.store.Update(p.Pipeline, q, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _, _ := f.runs.Get(run)
|
||||
var started []string
|
||||
f.s.Say = func(_ string, body any) error {
|
||||
if ps, ok := body.(seat.PipelineStarted); ok {
|
||||
if r, _, _ := f.runs.Get(run); r.Ask != before.Ask || r.Ended == nil {
|
||||
t.Errorf("pipeline-started said after the run was started again (ask %s, was %s)", r.Ask, before.Ask)
|
||||
}
|
||||
started = append(started, ps.Pipeline)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
|
||||
return Half{Entries: map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
|
||||
}, Acts: &actLog{}, RevisionHeld: func(string, int64) error { return nil }}
|
||||
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(started) != 1 || started[0] != p.Pipeline {
|
||||
t.Errorf("pipeline-started at the reopening = %v, want %s once", started, p.Pipeline)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(run); r.Ask != a.Result.Ask || r.Ended != nil {
|
||||
t.Errorf("the rerun's run = %+v; want the new ask running", r)
|
||||
}
|
||||
})
|
||||
// A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read,
|
||||
// carrying that reader, never answering the change as an error nor putting back the earlier readers.
|
||||
t.Run("a rerun writes its run from the latest read", func(t *testing.T) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -29,8 +30,15 @@ type Verbs struct {
|
||||
Acts views.HandActs
|
||||
// Runs answers the read verb test-runs, answered by the process that holds test-runs-cap-mib (A-470i).
|
||||
Runs RunsRead
|
||||
// RevisionHeld says whether a branch's catalog revision is still kept, read through the catalog seat's verb revision
|
||||
// at the time of the ask: nil when it is, an error wrapping ErrRevisionGone when it is removed (design 47 H2 H-A-479
|
||||
// addendum (2)).
|
||||
RevisionHeld func(branch string, revision int64) error
|
||||
}
|
||||
|
||||
// ErrRevisionGone is a catalog revision the catalog no longer keeps (H-A-479 addendum (2)).
|
||||
var ErrRevisionGone = errors.New("the catalog revision is gone")
|
||||
|
||||
func (v Verbs) now() time.Time { return v.Hearer.now() }
|
||||
|
||||
func (v Verbs) record(verb, what, why string) error {
|
||||
@@ -275,13 +283,89 @@ func (v Verbs) EndAgain(_ string, in seat.EndAgainInput) (seat.EndAgainAnswer, e
|
||||
// the ask saying it is a rerun and the caller the bus proved; a failed pipeline runs again and ends once more with its end
|
||||
// count one higher (§325), an approval never asked waiting again. A review waits again; an approval is asked again by
|
||||
// `ask-again`.
|
||||
//
|
||||
// A rerun of a step in an ended delivering pipeline reopens it: its revision is read first and the rerun refused by name,
|
||||
// nothing asked, when the catalog removed it; once reopened its pipeline-started is said again, so the indexer pins its
|
||||
// revision again until its next end (design 47 H2 H-A-479 addendum (2)).
|
||||
func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error) {
|
||||
if in.Step == seat.StepResultStepApproval {
|
||||
return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun")
|
||||
}
|
||||
reopens, err := v.reopening(in.Pipeline)
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
return v.rerun(caller, in, reopens)
|
||||
}
|
||||
|
||||
// sayReopened says pipeline-started again for p, written reopened by a rerun, before its step is asked again, so the
|
||||
// indexer pins its revision again before anything reads it (H-A-479 addendum (2)).
|
||||
func (v Verbs) sayReopened(p seat.Pipeline, reopens bool) {
|
||||
if !reopens || v.Starter == nil || p.Ended != nil {
|
||||
return
|
||||
}
|
||||
v.Starter.sayStartedPipeline(p, v.now())
|
||||
v.Starter.logf("%s reopened by a rerun: its pipeline-started is said again, so its revision %d is pinned again (H-A-479)", p.Pipeline, p.Revision)
|
||||
}
|
||||
|
||||
// reopening says whether a rerun of pipeline key reopens an ended delivering pipeline, refusing it when its revision is
|
||||
// gone (H-A-479 addendum (2)). A pipeline not read, a run's, or one the rerun refuses anyway is left to the rerun.
|
||||
func (v Verbs) reopening(key string) (bool, error) {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil || p.Ended == nil || p.Run != "" || p.Revision <= 0 ||
|
||||
p.Outcome == seat.PipelineOutcomeCancelled || p.Outcome == seat.PipelineOutcomeSuperseded {
|
||||
return false, nil
|
||||
}
|
||||
// A reopening never builds beside an open pipeline carrying one of its movers at the same revision (H-A-479 third
|
||||
// addendum).
|
||||
if open, err := v.openBeside(p); err != nil {
|
||||
return false, err
|
||||
} else if open != "" {
|
||||
return false, fmt.Errorf("the pipeline %s ended and is not reopened: its pipeline %s is running: nothing was asked", key, open)
|
||||
}
|
||||
if v.RevisionHeld != nil {
|
||||
if err := v.RevisionHeld(p.Branch, p.Revision); errors.Is(err, ErrRevisionGone) {
|
||||
return false, fmt.Errorf("the pipeline %s ended and its revision %d is gone; start the mover again at the head: nothing was asked (%v)", key, p.Revision, err)
|
||||
} else if err != nil {
|
||||
return false, fmt.Errorf("the pipeline %s ended and its revision %d is not read, so it is not reopened: nothing was asked: %w", key, p.Revision, err)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// openBeside is the key of another delivering pipeline of p's branch and revision, not ended, carrying one of p's movers it
|
||||
// did not leave; empty when none (H-A-479 third addendum).
|
||||
func (v Verbs) openBeside(p seat.Pipeline) (string, error) {
|
||||
pkeys, err := branchKeys(v.Hearer.Store, p.Branch)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the pipelines of %s are not listed, so %s is not reopened: nothing was asked: %w", p.Branch, p.Pipeline, err)
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
if key == p.Pipeline {
|
||||
continue
|
||||
}
|
||||
o, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the pipeline %s is not read, so %s is not reopened: nothing was asked: %w", key, p.Pipeline, err)
|
||||
}
|
||||
if o.Run != "" || o.Ended != nil || o.Revision != p.Revision ||
|
||||
(o.Outcome != seat.PipelineOutcomeRunning && o.Outcome != seat.PipelineOutcomeWaiting) {
|
||||
continue
|
||||
}
|
||||
for _, m := range p.Movers {
|
||||
if Moves(o, m.Module) && !slices.Contains(o.Superseded, m.Module) {
|
||||
return key, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// rerun asks one step again (Rerun).
|
||||
func (v Verbs) rerun(caller string, in seat.RerunInput, reopens bool) (seat.RerunAnswer, error) {
|
||||
if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil {
|
||||
if i, err := stepOf(p, in.Module, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) {
|
||||
return v.rerunTest(caller, in, p.Steps[i])
|
||||
return v.rerunTest(caller, in, p.Steps[i], reopens)
|
||||
}
|
||||
}
|
||||
var ask *Asked
|
||||
@@ -332,6 +416,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
v.sayReopened(p, reopens)
|
||||
if ask != nil {
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm, v.Starter.logf); err != nil {
|
||||
@@ -349,7 +434,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one
|
||||
// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set
|
||||
// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass.
|
||||
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) {
|
||||
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult, reopens bool) (seat.RerunAnswer, error) {
|
||||
h := v.Hearer
|
||||
run := sharedRun(st)
|
||||
if open(st) {
|
||||
@@ -430,6 +515,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
v.sayReopened(p, reopens)
|
||||
switch {
|
||||
case join || read:
|
||||
v.Starter.joinRun(run, reader)
|
||||
@@ -649,10 +735,17 @@ func (v Verbs) Run(caller string, in seat.RunInput) (seat.RunAnswer, error) {
|
||||
|
||||
// Start starts by hand a delivering pipeline for the head of a watched branch the runner's catalog half holds, recorded in
|
||||
// the hand-act log first (§306, §394): the pipeline of that branch's source revision and the module, as a take-in would
|
||||
// start it. A head whose pipeline is written already starts nothing and is refused, naming the pipeline.
|
||||
// start it. A head whose pipeline is written already starts nothing and is refused, naming the pipeline. With modules it
|
||||
// starts the named modules again at the head instead (startAgain, H-A-479); exactly one of module or modules is given.
|
||||
func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) {
|
||||
if in.Module != "" && len(in.Modules) > 0 {
|
||||
return seat.StartAnswer{}, errors.New("start names the module or the modules, not both: nothing was started")
|
||||
}
|
||||
if in.Branch != "" && len(in.Modules) > 0 {
|
||||
return v.startAgain(in)
|
||||
}
|
||||
if in.Branch == "" || in.Module == "" {
|
||||
return seat.StartAnswer{}, errors.New("start names the watched branch and the module")
|
||||
return seat.StartAnswer{}, errors.New("start names the watched branch and the module or the modules")
|
||||
}
|
||||
if v.Starter == nil || v.View == nil {
|
||||
return seat.StartAnswer{}, errors.New("the runner cannot start a pipeline: it holds no starter or no catalog half")
|
||||
@@ -681,6 +774,117 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) {
|
||||
return seat.StartAnswer{Pipeline: key}, nil
|
||||
}
|
||||
|
||||
// startAgain starts by hand, recorded in the hand-act log first, a pipeline at the head of a watched branch carrying the
|
||||
// named modules whose newest entry, the head's, has no passed pipeline (a failed, cut or superseded one), with the head's
|
||||
// revision as its candidate and the take-in's tiers (design 47 H2 H-A-479): keyed `<branch>.<revision>.again.<n>`, n from 1,
|
||||
// taken by writing on absence (addendum (1)). Its pipeline-started pins the head's revision as a take-in's does, and its
|
||||
// versions are reserved at its build steps as a take-in's are. A module the head holds no entry of, or whose newest entry
|
||||
// passed in a pipeline, is refused by name, nothing started.
|
||||
func (v Verbs) startAgain(in seat.StartInput) (seat.StartAnswer, error) {
|
||||
if v.Starter == nil || v.View == nil || v.Hearer == nil || v.Hearer.Store == nil {
|
||||
return seat.StartAnswer{}, errors.New("the runner cannot start a pipeline again: it holds no starter, no catalog half or no state pipelines")
|
||||
}
|
||||
half, ok := v.View(in.Branch)
|
||||
if !ok || half.Source <= 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s is not held at a source revision yet: nothing was started", in.Branch)
|
||||
}
|
||||
movers := make([]seat.Mover, 0, len(in.Modules))
|
||||
var absent []string
|
||||
for _, m := range in.Modules {
|
||||
e, ok := half.Entries[m]
|
||||
if !ok {
|
||||
absent = append(absent, m)
|
||||
continue
|
||||
}
|
||||
if slices.ContainsFunc(movers, func(x seat.Mover) bool { return x.Module == m }) {
|
||||
continue
|
||||
}
|
||||
movers = append(movers, seat.Mover{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
|
||||
}
|
||||
if len(absent) > 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the head of %s (revision %d) holds no entry of %s: nothing was started", in.Branch, half.Revision, strings.Join(absent, ", "))
|
||||
}
|
||||
passed, running, err := v.heldEntries(in.Branch, movers)
|
||||
if err != nil {
|
||||
return seat.StartAnswer{}, err
|
||||
}
|
||||
if len(passed) > 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the newest entry of %s passed already: nothing was started", strings.Join(passed, ", "))
|
||||
}
|
||||
if len(running) > 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the newest entry of %s: nothing was started", strings.Join(running, ", "))
|
||||
}
|
||||
movers = Tiers(movers, half.Entries, v.Starter.logf)
|
||||
names := make([]string, 0, len(movers))
|
||||
for _, m := range movers {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
prefix := AgainPipelineKey(in.Branch, half.Revision, 0)
|
||||
prefix = prefix[:len(prefix)-1]
|
||||
held, err := v.Hearer.Store.Keys(prefix + "*")
|
||||
if err != nil {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the pipelines started again at revision %d of %s are not listed: nothing was started: %w", half.Revision, in.Branch, err)
|
||||
}
|
||||
n := 1
|
||||
for _, k := range held {
|
||||
if i, err := strconv.Atoi(strings.TrimPrefix(k, prefix)); err == nil && i >= n {
|
||||
n = i + 1
|
||||
}
|
||||
}
|
||||
if err := v.record(seat.VerbStart, fmt.Sprintf("started %s again at the head of %s by hand", strings.Join(names, ", "), in.Branch), in.Why); err != nil {
|
||||
return seat.StartAnswer{}, err
|
||||
}
|
||||
// The next n is taken without a bound: each refusal means the key exists (H-A-479 second addendum).
|
||||
for ; ; n++ {
|
||||
key := AgainPipelineKey(in.Branch, half.Revision, n)
|
||||
started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries,
|
||||
Key: key, Movers: movers})
|
||||
if err != nil {
|
||||
return seat.StartAnswer{}, err
|
||||
}
|
||||
if started {
|
||||
v.Starter.logf("%s starts %s again at the head by hand (H-A-479)", key, strings.Join(names, ", "))
|
||||
return seat.StartAnswer{Pipeline: key}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// heldEntries are the movers whose module, at the mover's fingerprint, a delivering pipeline of branch carries and did not
|
||||
// leave: passed, those a passed pipeline holds (a newest entry that passed is not started again, H-A-479); running, those a
|
||||
// pipeline not ended holds, each said "<module>: its pipeline <key> is running", so two pipelines never build one entry side
|
||||
// by side (H-A-479 second addendum).
|
||||
func (v Verbs) heldEntries(branch string, movers []seat.Mover) (passed, running []string, err error) {
|
||||
pkeys, err := branchKeys(v.Hearer.Store, branch)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the pipelines of %s are not listed, so nothing was started: %w", branch, err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the pipeline %s is not read, so nothing was started: %w", key, err)
|
||||
}
|
||||
open := p.Ended == nil && (p.Outcome == seat.PipelineOutcomeRunning || p.Outcome == seat.PipelineOutcomeWaiting)
|
||||
if p.Run != "" || (p.Outcome != seat.PipelineOutcomePassed && !open) {
|
||||
continue
|
||||
}
|
||||
for _, m := range movers {
|
||||
if seen[m.Module] || slices.Contains(p.Superseded, m.Module) {
|
||||
continue
|
||||
}
|
||||
if pm, ok := MoverOf(p, m.Module); ok && pm.Fingerprint == m.Fingerprint {
|
||||
seen[m.Module] = true
|
||||
if open {
|
||||
running = append(running, m.Module+": its pipeline "+key+" is running")
|
||||
} else {
|
||||
passed = append(passed, m.Module+" (in "+key+")")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return passed, running, nil
|
||||
}
|
||||
|
||||
// AskAgain asks a declined approval step once more (§306, §36), recorded in the hand-act log first: the step waits again,
|
||||
// marked asked again by the caller the bus proved, and is asked at once when the pipeline's every other required step
|
||||
// passed, under an ask of its own (the earlier ask's decision ended it).
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -423,3 +425,180 @@ func TestAValidateRerunMovesEveryMoversValidateAndReopensItsCompositions(t *test
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-479: start with modules starts a pipeline at the head keyed <branch>.<revision>.again.<n> carrying the named modules
|
||||
// whose newest entry has no passed pipeline, at the take-in's tiers; a module whose newest entry passed is refused by name,
|
||||
// nothing started; exactly one of module or modules is given; a second start takes the next n.
|
||||
func TestStartWithModulesStartsAFailedMoverAgainAtTheHead(t *testing.T) {
|
||||
v, store, _, _, acts, _ := verbsFixture()
|
||||
made := created{}
|
||||
var started []seat.PipelineStarted
|
||||
v.Starter.Pipelines = made
|
||||
v.Starter.Say = func(_ string, body any) error {
|
||||
if ps, ok := body.(seat.PipelineStarted); ok {
|
||||
started = append(started, ps)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, entries := placeFixture()
|
||||
app := entries["app"]
|
||||
app.Fp, app.Commit, app.Repository = "src2:aa", "c1", "o/app"
|
||||
entries["app"] = app
|
||||
entries["lib"] = catalog.CatalogEntry{Module: "lib", Fp: "src2:bb", Commit: "c2", Repository: "o/lib"}
|
||||
v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 9, Entries: entries}, true }
|
||||
// The fixture's main.5 moves app at src2:aa and failed; lib's newest entry passed in main.8.
|
||||
e := store[pkey]
|
||||
e.p.Outcome = seat.PipelineOutcomeFailed
|
||||
store[pkey] = e
|
||||
store["main.8"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: "main.8", Branch: "main", Source: 6, Revision: 8, Outcome: seat.PipelineOutcomePassed,
|
||||
Movers: []seat.Mover{{Module: "lib", Fingerprint: "src2:bb"}, {Module: "app", Fingerprint: "src2:aa"}}, Superseded: []string{"app"}}, 1}
|
||||
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Modules: []string{"app"}, Why: "x"}); err == nil {
|
||||
t.Error("a start naming both module and modules was taken")
|
||||
}
|
||||
// A pipeline still running at app's head entry refuses the start: two pipelines never build one entry side by side.
|
||||
store["main.7"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: "main.7", Branch: "main", Source: 6, Revision: 7, Outcome: seat.PipelineOutcomeRunning,
|
||||
Movers: []seat.Mover{{Module: "app", Fingerprint: "src2:aa"}}}, 1}
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "x"}); err == nil || !strings.Contains(err.Error(), "its pipeline main.7 is running") {
|
||||
t.Errorf("a start of app while main.7 runs it = %v; want refused naming main.7", err)
|
||||
}
|
||||
delete(store, "main.7")
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app", "lib"}, Why: "x"}); err == nil || !strings.Contains(err.Error(), "lib") {
|
||||
t.Errorf("a start naming lib, whose newest entry passed, = %v; want refused naming lib", err)
|
||||
}
|
||||
if len(made) != 0 {
|
||||
t.Fatalf("a refused start wrote %d pipelines", len(made))
|
||||
}
|
||||
a, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "its pipeline failed at an unchanged entry"})
|
||||
if err != nil || a.Pipeline != "main.9.again.1" || made["main.9.again.1"] == nil {
|
||||
t.Fatalf("start with modules = %+v, %v; written %v", a, err, len(made))
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(made["main.9.again.1"], &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Revision != 9 || p.Source != 7 || len(p.Movers) != 1 || p.Movers[0].Module != "app" || p.Movers[0].Fingerprint != "src2:aa" || p.Movers[0].Tier != 1 {
|
||||
t.Errorf("the pipeline = %+v, want app at src2:aa, tier 1, at revision 9 of source 7", p)
|
||||
}
|
||||
if len(started) != 1 || started[0].Pipeline != "main.9.again.1" || started[0].Revision != 9 {
|
||||
t.Errorf("pipeline-started = %+v, want main.9.again.1 at revision 9 (its revision pinned)", started)
|
||||
}
|
||||
// The next start at the same head, once main.9.again.1 failed, takes the next n.
|
||||
at := time.Unix(3, 0)
|
||||
p.Outcome, p.Ended = seat.PipelineOutcomeFailed, &at
|
||||
store["main.9.again.1"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{p, 1}
|
||||
if a, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "again"}); err != nil || a.Pipeline != "main.9.again.2" {
|
||||
t.Errorf("a second start with modules = %+v, %v; want main.9.again.2", a, err)
|
||||
}
|
||||
if len(*acts) != 2 {
|
||||
t.Errorf("hand acts %v, want the two starts recorded", *acts)
|
||||
}
|
||||
// main.9.again.1 passed: its version is written into it as into any pipeline of its source, and a further start of app is
|
||||
// refused naming it, read through the branch's key forms.
|
||||
p.Outcome = seat.PipelineOutcomePassed
|
||||
store["main.9.again.1"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{p, 1}
|
||||
if wrote, err := v.Hearer.Released(catalog.VersionReleasedEvent{Module: "app", Version: "0.1.2", Source: 7}, "main"); err != nil || wrote != 1 ||
|
||||
store["main.9.again.1"].p.Movers[0].Version != "0.1.2" {
|
||||
t.Errorf("the release of app 0.1.2 wrote %d (%v), the again pipeline's mover %+v; want its version written", wrote, err, store["main.9.again.1"].p.Movers)
|
||||
}
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "x"}); err == nil || !strings.Contains(err.Error(), "main.9.again.1") {
|
||||
t.Errorf("a start of app after main.9.again.1 passed = %v; want refused naming it", err)
|
||||
}
|
||||
if !IsStepsKey(StepsKey("main.9.again.1", "app")) {
|
||||
t.Error("a mover key of an again pipeline is not read as a mover's steps")
|
||||
}
|
||||
// H-A-485: the mover keys are read by the one parse of the seat's key forms; a pipeline key of no form (its n or its
|
||||
// revision no number) is no mover's steps.
|
||||
for _, k := range []string{"steps.main.9.again.x.app", "steps.main.r9.app", "steps.main.9.again.0.app"} {
|
||||
if IsStepsKey(k) {
|
||||
t.Errorf("%s, of no pipeline key form, is read as a mover's steps", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-479 addendum (2): a rerun of a step in an ended pipeline pins its revision again at the reopening (its pipeline-started
|
||||
// said again), and is refused by name, nothing asked, when the revision is already removed.
|
||||
func TestARerunOfAnEndedPipelinePinsItsRevisionAgainOrIsRefusedWhenItIsGone(t *testing.T) {
|
||||
v, store, _, asks, _, _ := verbsFixture()
|
||||
var started []seat.PipelineStarted
|
||||
v.Starter.Say = func(_ string, body any) error {
|
||||
if ps, ok := body.(seat.PipelineStarted); ok {
|
||||
if len(asks.asks) != 0 {
|
||||
t.Errorf("pipeline-started said after the step was asked again (%d asks): its revision is pinned too late", len(asks.asks))
|
||||
}
|
||||
started = append(started, ps)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
e := store[pkey]
|
||||
at := time.Unix(2, 0)
|
||||
e.p.Revision, e.p.Outcome, e.p.Ended = 5, seat.PipelineOutcomeFailed, &at
|
||||
for i := range e.p.Steps {
|
||||
if e.p.Steps[i].Artifact == "app:program" && e.p.Steps[i].Step == "lint" {
|
||||
e.p.Steps[i].Outcome = seat.StepResultOutcomeFailed
|
||||
}
|
||||
}
|
||||
store[pkey] = e
|
||||
gone := true
|
||||
v.RevisionHeld = func(branch string, revision int64) error {
|
||||
if gone {
|
||||
return fmt.Errorf("revision %s.%d: no such object: %w", branch, revision, ErrRevisionGone)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
in := seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "retry"}
|
||||
// H-A-479 third addendum: another open pipeline carrying app at the same revision refuses the reopening.
|
||||
store["main.5.again.1"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: "main.5.again.1", Branch: "main", Source: 5, Revision: 5, Outcome: seat.PipelineOutcomeRunning,
|
||||
Movers: []seat.Mover{{Module: "app", Fingerprint: "src2:aa"}}}, 1}
|
||||
gone = false
|
||||
if _, err := v.Rerun("operator", in); err == nil || !strings.Contains(err.Error(), "its pipeline main.5.again.1 is running") {
|
||||
t.Fatalf("a reopening beside main.5.again.1 = %v; want refused naming it", err)
|
||||
}
|
||||
if len(asks.asks) != 0 || store[pkey].p.Outcome != seat.PipelineOutcomeFailed {
|
||||
t.Fatalf("a refused reopening asked %d, left the pipeline %s", len(asks.asks), store[pkey].p.Outcome)
|
||||
}
|
||||
delete(store, "main.5.again.1")
|
||||
gone = true
|
||||
if _, err := v.Rerun("operator", in); err == nil || !strings.Contains(err.Error(), "its revision 5 is gone; start the mover again at the head") {
|
||||
t.Fatalf("a rerun of an ended pipeline whose revision is gone = %v; want refused by name", err)
|
||||
}
|
||||
if len(asks.asks) != 0 || store[pkey].p.Outcome != seat.PipelineOutcomeFailed || len(started) != 0 {
|
||||
t.Fatalf("a refused rerun asked %d, left the pipeline %s, said %d starts", len(asks.asks), store[pkey].p.Outcome, len(started))
|
||||
}
|
||||
gone = false
|
||||
if _, err := v.Rerun("operator", in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(started) != 1 || started[0].Pipeline != pkey || started[0].Revision != 5 {
|
||||
t.Errorf("pipeline-started at the reopening = %+v, want %s at revision 5 (its revision pinned again)", started, pkey)
|
||||
}
|
||||
// A rerun of a running pipeline reopens nothing and says no start.
|
||||
e = store[pkey]
|
||||
for i := range e.p.Steps {
|
||||
if e.p.Steps[i].Artifact == "app:program" && e.p.Steps[i].Step == "unit" {
|
||||
e.p.Steps[i].Outcome = seat.StepResultOutcomeFailed
|
||||
}
|
||||
}
|
||||
store[pkey] = e
|
||||
if _, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "unit", Artifact: "app:program", Why: "x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(started) != 1 {
|
||||
t.Errorf("a rerun of a running pipeline said pipeline-started again: %+v", started)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,7 +237,8 @@ func runSettingsOf(raw []byte) bool {
|
||||
}
|
||||
if len(m.Config) != len(want)+len(words) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
|
||||
m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) ||
|
||||
m.Config["pipelines-at-once"]["default"] != float64(1) || m.Config["pipelines-waiting-cap"]["default"] != float64(64) {
|
||||
m.Config["pipelines-at-once"]["default"] != float64(1) || m.Config["pipelines-waiting-cap"]["default"] != float64(64) ||
|
||||
m.Config["widens-read-bound"]["default"] != "30s" {
|
||||
return false
|
||||
}
|
||||
env := map[string]bool{}
|
||||
|
||||
@@ -138,9 +138,9 @@
|
||||
},
|
||||
"widens-read-bound": {
|
||||
"type": "string",
|
||||
"description": "how long one read of a pipeline end's widens (the catalog's revision, the seat holders, the grants part) waits for its answer before it is cut and said could not run, a Go duration such as 5s (design 47 G A-472, A-472a).",
|
||||
"description": "how long one read of a pipeline end's widens (the catalog's revision, the seat holders, the grants part) waits for its answer before it is cut and said could not run, a Go duration such as 30s (design 47 G A-472, A-472a, H2 H-A-478: long enough that a cold grants-for is not cut).",
|
||||
"trusted": false,
|
||||
"default": "5s"
|
||||
"default": "30s"
|
||||
},
|
||||
"revisions-kept": {
|
||||
"type": "number",
|
||||
|
||||
@@ -13,6 +13,7 @@ require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -27,6 +27,10 @@ const (
|
||||
IdleAfter = 24 * time.Hour
|
||||
// FullPercent is the fill of the objects past which the receiver says they fill.
|
||||
FullPercent = 80
|
||||
// TakeOverAfter is how long a transfer's last part must lie behind it before another caller opening it with the
|
||||
// same digest takes it over (design 47 H2 H-A-481): a sender restarted mid-transfer goes on, a live one is not
|
||||
// pushed aside.
|
||||
TakeOverAfter = 2 * time.Minute
|
||||
)
|
||||
|
||||
// Want is one part an opening names: its digest (sha256:<hex>) and its size.
|
||||
@@ -117,12 +121,17 @@ type Receiver[M any] struct {
|
||||
// Bound reads the cap the objects' binding carries (BusCap): the bus's own bound, which then counts over Cap.
|
||||
// Optional; an error or a zero falls back to Cap.
|
||||
Bound func() (uint64, error)
|
||||
// MaxChunk, Idle and Full override MaxReceived, IdleAfter and FullPercent when set.
|
||||
// MaxChunk, Idle, TakeOver and Full override MaxReceived, IdleAfter, TakeOverAfter and FullPercent when set.
|
||||
MaxChunk int
|
||||
Idle time.Duration
|
||||
TakeOver time.Duration
|
||||
Full uint64
|
||||
// Holds says the holder already holds a part, which then counts as held at the open and is not sent. Optional.
|
||||
Holds func(t Transfer[M], digest string) bool
|
||||
// HoldsWhole says the holder holds the file itself the transfer brings (its manifest or reference at the name and
|
||||
// version its shape names), never from its parts alone (design 47 H2 H-A-481 second addendum): a reopen that lacks
|
||||
// nothing is then dropped without a second store. Optional: nil never drops, and the reopen finishes as usual.
|
||||
HoldsWhole func(t Transfer[M]) bool
|
||||
// Check is the caller's check of a whole transfer before anything is stored; its error is the refusal's reason.
|
||||
// Optional.
|
||||
Check func(t Transfer[M]) error
|
||||
@@ -172,6 +181,13 @@ func (r *Receiver[M]) CapNow() uint64 {
|
||||
return r.Cap
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) takeOver() time.Duration {
|
||||
if r.TakeOver > 0 {
|
||||
return r.TakeOver
|
||||
}
|
||||
return TakeOverAfter
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) fullPct() uint64 {
|
||||
if r.Full > 0 {
|
||||
return r.Full
|
||||
@@ -180,8 +196,12 @@ func (r *Receiver[M]) fullPct() uint64 {
|
||||
}
|
||||
|
||||
// Open opens a transfer for opener, or opens it again: a transfer already open answers what it lacks, so a cut transfer
|
||||
// goes on from there, and only its opener may open it again. A new transfer that would pass the cap is refused; parts
|
||||
// the holder holds count as held; when nothing is lacked the transfer ends at once.
|
||||
// goes on from there. A transfer is finished or taken over, never left open (design 47 H2 H-A-481 and its addendum):
|
||||
// one opened again with nothing lacked (its finish was cut) is finished, by its opener or by another caller with the
|
||||
// same digest, once each part taken in it is read back and hashes to its digest (a part that does not is lacked again),
|
||||
// and dropped without a second store when the holder already holds the file itself (HoldsWhole); one that lacks a part is another
|
||||
// caller's only once its last part is older than TakeOver, and is refused to it before. A new transfer that would pass the cap is refused; parts the holder holds count as held; when nothing is
|
||||
// lacked the transfer ends at once, its idle bound armed before the finish, so a cut finish never leaves it open.
|
||||
func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) {
|
||||
switch {
|
||||
case strings.TrimSpace(id) == "" || strings.ContainsAny(id, "/. *>"):
|
||||
@@ -197,8 +217,33 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
if t, found, err := r.Transfers.Get(id); err != nil {
|
||||
return Received{}, err
|
||||
} else if found {
|
||||
if t.Opener != opener {
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id)
|
||||
if len(t.Lacks()) == 0 {
|
||||
if r.heldWhole(t) {
|
||||
// The cut came after the store: the holder holds the file, so it is not stored a second time.
|
||||
if _, err := r.Drop(id); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s is stored already and cannot be dropped: %w", id, err)
|
||||
}
|
||||
return Received{ID: id, Ended: true}, nil
|
||||
}
|
||||
if !r.verify(&t) {
|
||||
return r.finishArmed(t)
|
||||
}
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err)
|
||||
}
|
||||
r.tell(t.FillAtOpen)
|
||||
}
|
||||
switch {
|
||||
case len(t.Lacks()) == 0:
|
||||
return r.finishArmed(t)
|
||||
case t.Opener != opener && r.now().Sub(t.Last) < r.takeOver():
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller, whose last part came %s ago (taken over after %s)",
|
||||
opener, id, r.now().Sub(t.Last).Round(time.Second), r.takeOver())
|
||||
case t.Opener != opener:
|
||||
t.Opener = opener
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be taken over by %s: %w", id, opener, err)
|
||||
}
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
@@ -227,11 +272,53 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
}
|
||||
r.tell(used)
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
return r.finishArmed(t)
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
|
||||
// heldWhole says the holder already holds the file t brings (HoldsWhole), so a finish cut after its store stores nothing
|
||||
// again; parts held alone never say it (H-A-481 second addendum), since a publish cut before its store must still be
|
||||
// said as published or refused.
|
||||
func (r *Receiver[M]) heldWhole(t Transfer[M]) bool {
|
||||
return r.HoldsWhole != nil && r.HoldsWhole(t)
|
||||
}
|
||||
|
||||
// verify reads each part taken in t back from the objects and hashes it against its digest, as Take does at a part's
|
||||
// last chunk (H-A-481 addendum): a finish cut after its chunks were dropped reads them absent, which the runtime's
|
||||
// object read answers as empty. A part that fails has its chunks dropped and is lacked again. It answers whether any
|
||||
// part failed, t changed with it.
|
||||
func (r *Receiver[M]) verify(t *Transfer[M]) bool {
|
||||
failed := false
|
||||
for _, p := range t.Parts {
|
||||
if !t.Taken(p.Digest) || !slices.Contains(t.Held, p.Digest) {
|
||||
continue
|
||||
}
|
||||
whole, err := r.part(*t, p.Digest)
|
||||
if err == nil && "sha256:"+sum(whole) == p.Digest {
|
||||
continue
|
||||
}
|
||||
r.dropChunks(*t, p.Digest)
|
||||
delete(t.Chunks, p.Digest)
|
||||
t.Held = slices.DeleteFunc(t.Held, func(d string) bool { return d == p.Digest })
|
||||
failed = true
|
||||
}
|
||||
if failed {
|
||||
r.reanchor(t)
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
// finishArmed finishes a transfer that lacks nothing at an open, its idle bound armed first (H-A-481): a finish cut
|
||||
// by a restart leaves a kept transfer its key still drops, and the next open finishes it. A key not written fails the
|
||||
// open loudly before the finish, the transfer kept for the next open.
|
||||
func (r *Receiver[M]) finishArmed(t Transfer[M]) (Received, error) {
|
||||
if err := r.arm(t.ID); err != nil {
|
||||
return Received{}, err
|
||||
}
|
||||
return Received{ID: t.ID, Ended: true}, r.finish(t)
|
||||
}
|
||||
|
||||
// arm bounds an open transfer by its idle key. A key that cannot be written fails the open loudly (the transfer is
|
||||
// kept, so the opener's next open of it answers what is held and arms it again).
|
||||
func (r *Receiver[M]) arm(id string) error {
|
||||
|
||||
@@ -289,6 +289,106 @@ func aHolderTakesAFileInParts(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// Design 47 H2 H-A-481: a transfer is finished or taken over, never left open. A finish cut by a restart leaves the
|
||||
// transfer kept with every part held and its chunks in the objects (cut): the opener's next open, or another caller's
|
||||
// with the same digest, finishes it; a transfer that lacks a part is another caller's once its last part is older
|
||||
// than TakeOverAfter, refused to it before; and a transfer that lacks nothing at its open is armed before its finish.
|
||||
t.Run("a transfer is finished or taken over, never left open", func(t *testing.T) {
|
||||
var order []string
|
||||
pkg.Arm = func(id string, _ time.Duration) error { order = append(order, "arm "+id); return nil }
|
||||
store := pkg.Store
|
||||
defer func() { pkg.Arm, pkg.Holds, pkg.HoldsWhole, pkg.Store = nil, nil, nil, store }()
|
||||
// cut leaves c1 as a cut finish does: "whole" all chunks there, "lacked" a part short, "gone" its chunks dropped
|
||||
// before the cut, "wrong" a chunk that does not hash.
|
||||
cut := func(id, how string) {
|
||||
tr := Transfer[packageShape]{ID: id, Opener: "builder", Parts: []Want{part}, Chunks: map[string]int{part.Digest: 1},
|
||||
Held: []string{part.Digest}, Opened: now, Last: now, Shape: packageShape{"m", "1"}}
|
||||
switch how {
|
||||
case "whole":
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = blob
|
||||
case "lacked":
|
||||
tr.Held = nil
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = blob[:1000]
|
||||
case "wrong":
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = []byte("other")
|
||||
}
|
||||
if err := pkg.Transfers.Put(tr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name, opener, how string
|
||||
after, takeOver time.Duration
|
||||
storeHolds string // "parts": Holds answers every part; "file": HoldsWhole answers the file
|
||||
want string // "stored", "dropped", "lacked again", "lacks" (taken over) or the refusal's words
|
||||
}{
|
||||
{name: "the opener opens a cut finish again", opener: "builder", how: "whole", want: "stored"},
|
||||
{name: "another caller opens a cut finish", opener: "other", how: "whole", want: "stored"},
|
||||
{name: "a cut after the chunks were dropped", opener: "builder", how: "gone", want: "lacked again"},
|
||||
{name: "a cut finish whose chunk does not hash", opener: "builder", how: "wrong", want: "lacked again"},
|
||||
{name: "a cut after the store: the holder holds the file", opener: "builder", how: "whole", storeHolds: "file", want: "dropped"},
|
||||
{name: "a cut before the store: the holder holds the parts, not the file", opener: "builder", how: "whole", storeHolds: "parts", want: "stored"},
|
||||
{name: "another caller while the opener sends", opener: "other", how: "lacked", after: time.Minute, want: "another caller, whose last part came 1m0s ago"},
|
||||
{name: "another caller once the last part is two minutes old", opener: "other", how: "lacked", after: TakeOverAfter, want: "lacks"},
|
||||
{name: "the receiver's own takeover bound", opener: "other", how: "lacked", after: time.Minute, takeOver: 30 * time.Second, want: "lacks"},
|
||||
} {
|
||||
pkgLog = nil
|
||||
pkg.TakeOver = c.takeOver
|
||||
pkg.Holds, pkg.HoldsWhole = nil, nil
|
||||
switch c.storeHolds {
|
||||
case "parts":
|
||||
pkg.Holds = func(Transfer[packageShape], string) bool { return true }
|
||||
case "file":
|
||||
pkg.HoldsWhole = func(tr Transfer[packageShape]) bool { return tr.Shape == packageShape{"m", "1"} }
|
||||
}
|
||||
cut("c1", c.how)
|
||||
now = now.Add(c.after)
|
||||
got, err := pkg.Open("c1", c.opener, []Want{part}, packageShape{"m", "1"})
|
||||
kept, open, _ := pkg.Transfers.Get("c1")
|
||||
switch c.want {
|
||||
case "stored":
|
||||
if err != nil || !got.Ended || open || len(pkgLog) != 1 || pkgLog[0].outcome != "stored" || string(pkgLog[0].parts[part.Digest]) != string(blob) {
|
||||
t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it finished and stored", c.name, got, err, open, pkgLog)
|
||||
}
|
||||
case "dropped":
|
||||
if err != nil || !got.Ended || open || len(pkgLog) != 0 {
|
||||
t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it dropped, not stored again", c.name, got, err, open, pkgLog)
|
||||
}
|
||||
case "lacked again":
|
||||
if err != nil || got.Ended || !open || len(pkgLog) != 0 || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) || kept.Taken(part.Digest) {
|
||||
t.Errorf("%s: open = %+v, %v, log %+v, kept %+v; want the part lacked again, nothing stored", c.name, got, err, pkgLog, kept)
|
||||
}
|
||||
_, _ = pkg.Drop("c1")
|
||||
case "lacks":
|
||||
if err != nil || got.Ended || !open || kept.Opener != c.opener || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) {
|
||||
t.Errorf("%s: open = %+v, %v, opener %q; want it taken over by %s, lacking the part", c.name, got, err, kept.Opener, c.opener)
|
||||
}
|
||||
if r, err := pkg.Take("c1", c.opener, part.Digest, 1, true, blob[1000:], nil); err != nil || !r.Ended {
|
||||
t.Errorf("%s: the new opener's last chunk = %+v, %v", c.name, r, err)
|
||||
}
|
||||
default:
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) || !open || kept.Opener != "builder" {
|
||||
t.Errorf("%s: open = %v, opener %q; want refused with %q, the transfer kept", c.name, err, kept.Opener, c.want)
|
||||
}
|
||||
_, _ = pkg.Drop("c1")
|
||||
}
|
||||
if len(objects) != 0 {
|
||||
t.Errorf("%s: %d objects left", c.name, len(objects))
|
||||
}
|
||||
}
|
||||
pkg.TakeOver = 0
|
||||
order = nil
|
||||
pkg.Holds = func(Transfer[packageShape], string) bool { return true }
|
||||
pkg.Store = func(tr Transfer[packageShape], _ map[string][]byte) error {
|
||||
order = append(order, "store "+tr.ID)
|
||||
return nil
|
||||
}
|
||||
if got, err := pkg.Open("c2", "builder", []Want{part}, packageShape{"m", "1"}); err != nil || !got.Ended ||
|
||||
!reflect.DeepEqual(order, []string{"arm c2", "store c2"}) {
|
||||
t.Errorf("a transfer lacking nothing at its open = %+v, %v, %v; want its idle bound armed before its finish", got, err, order)
|
||||
}
|
||||
})
|
||||
|
||||
// An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the
|
||||
// rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the
|
||||
// binding carries (Bound) counts over Cap.
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// Package pipelinekey reads the key of a pipeline in the seat mesh-pipeline-runner's state pipelines, in every key form the
|
||||
// seat defines, through the seat's contract (design 47 H2 H-A-485): one parse, used by the runner, the assigner and the view, so
|
||||
// a pipeline started again at the head (`<branch>.<revision>.again.<n>`, H-A-479) or an unfinished mover's own pipeline
|
||||
// (`<branch>.<revision>.unfinished.<module>`, H-A-398) is read as a take-in's is. A key of no form the seat defines is refused
|
||||
// with an error, never read as an empty one.
|
||||
package pipelinekey
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The key forms of a pipeline in the state pipelines (the seat's key forms, design 47 H §746, H-A-213 (2), H-A-398, H-A-479).
|
||||
const (
|
||||
FormRevision = "<branch>.<revision>"
|
||||
FormRun = "<branch>.run.<run>"
|
||||
FormPublished = "<branch>.<source>.published.<package>.<version>"
|
||||
FormUnfinished = "<branch>.<revision>.unfinished.<module>"
|
||||
FormAgain = "<branch>.<revision>.again.<n>"
|
||||
)
|
||||
|
||||
// Key is a pipeline's key read: its form, its watched branch (unescaped), and the parts its form names: the revision (a
|
||||
// published form's source), the run, the package and version (escaped, as the key holds them), the unfinished mover's
|
||||
// module (unescaped) and the again form's n.
|
||||
type Key struct {
|
||||
Form string
|
||||
Branch string
|
||||
Revision int64
|
||||
Run string
|
||||
Package string
|
||||
Version string
|
||||
Module string
|
||||
N int
|
||||
}
|
||||
|
||||
// Parse reads key as a pipeline's key of one of the seat's forms; any other key is an error naming the forms.
|
||||
func Parse(key string) (Key, error) {
|
||||
parts := strings.Split(key, ".")
|
||||
var k Key
|
||||
var branch, revision, n, module string
|
||||
ok := false
|
||||
switch {
|
||||
case len(parts) == 2:
|
||||
branch, revision, ok = seat.ParsePipelinesByBranchRevisionKey(key)
|
||||
k.Form = FormRevision
|
||||
case len(parts) == 3 && parts[1] == "run":
|
||||
branch, k.Run, ok = seat.ParsePipelinesByBranchRunRunKey(key)
|
||||
k.Form = FormRun
|
||||
case len(parts) == 4 && parts[2] == "unfinished":
|
||||
branch, revision, module, ok = seat.ParsePipelinesByBranchRevisionUnfinishedModuleKey(key)
|
||||
k.Form = FormUnfinished
|
||||
case len(parts) == 4 && parts[2] == "again":
|
||||
branch, revision, n, ok = seat.ParsePipelinesByBranchRevisionAgainNKey(key)
|
||||
k.Form = FormAgain
|
||||
case len(parts) == 5 && parts[2] == "published":
|
||||
branch, revision, k.Package, k.Version, ok = seat.ParsePipelinesByBranchSourcePublishedPackageVersionKey(key)
|
||||
k.Form = FormPublished
|
||||
}
|
||||
if !ok {
|
||||
return Key{}, refused(key, "it is of no form the seat defines")
|
||||
}
|
||||
var err error
|
||||
if k.Branch, err = keys.UnescapeName(branch); err != nil || k.Branch == "" {
|
||||
return Key{}, refused(key, "its branch does not read")
|
||||
}
|
||||
if k.Form != FormRun {
|
||||
if k.Revision, err = strconv.ParseInt(revision, 10, 64); err != nil || k.Revision <= 0 {
|
||||
return Key{}, refused(key, "its revision is no number above 0")
|
||||
}
|
||||
}
|
||||
switch k.Form {
|
||||
case FormRun:
|
||||
if k.Run == "" {
|
||||
return Key{}, refused(key, "it names no run")
|
||||
}
|
||||
case FormUnfinished:
|
||||
if k.Module, err = keys.UnescapeName(module); err != nil || k.Module == "" {
|
||||
return Key{}, refused(key, "its module does not read")
|
||||
}
|
||||
case FormAgain:
|
||||
if k.N, err = strconv.Atoi(n); err != nil || k.N < 1 {
|
||||
return Key{}, refused(key, "its n is no number from 1")
|
||||
}
|
||||
case FormPublished:
|
||||
if k.Package == "" || k.Version == "" {
|
||||
return Key{}, refused(key, "it names no package or no version")
|
||||
}
|
||||
}
|
||||
return k, nil
|
||||
}
|
||||
|
||||
// refused is the error of a key Parse does not read.
|
||||
func refused(key, why string) error {
|
||||
return fmt.Errorf("%q is no key of a pipeline of the seat %s (%s): %s", key, seat.Seat,
|
||||
strings.Join([]string{FormRevision, FormRun, FormPublished, FormUnfinished, FormAgain}, ", "), why)
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package pipelinekey
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// H-A-485: every key form of a pipeline the seat mesh-pipeline-runner defines is read, the again form and the unfinished form as
|
||||
// a take-in's, its branch and module unescaped; a key of no form, or whose parts do not read, is refused naming why.
|
||||
func TestEveryPipelineKeyFormIsReadAndAnyOtherRefused(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
key string
|
||||
want Key
|
||||
why string
|
||||
}{
|
||||
{key: "main.265", want: Key{Form: FormRevision, Branch: "main", Revision: 265}},
|
||||
{key: "main.265.again.2", want: Key{Form: FormAgain, Branch: "main", Revision: 265, N: 2}},
|
||||
{key: "feat=2fx.12.unfinished.mesh-tools", want: Key{Form: FormUnfinished, Branch: "feat/x", Revision: 12, Module: "mesh-tools"}},
|
||||
{key: "main.run.r7", want: Key{Form: FormRun, Branch: "main", Run: "r7"}},
|
||||
{key: "main.40.published.lib.0=2e1=2e2", want: Key{Form: FormPublished, Branch: "main", Revision: 40, Package: "lib", Version: "0=2e1=2e2"}},
|
||||
{key: "main", why: "no form"},
|
||||
{key: "main.265.again", why: "no form"},
|
||||
{key: "main.265.later.2", why: "no form"},
|
||||
{key: "main.265.again.0", why: "its n"},
|
||||
{key: "main.265.again.x", why: "its n"},
|
||||
{key: "main.r1", why: "its revision"},
|
||||
{key: "main.0", why: "its revision"},
|
||||
{key: "ma=zz.5", why: "its branch"},
|
||||
{key: "main.5.unfinished.=zz", why: "its module"},
|
||||
{key: "main.5.6.7.8.9", why: "no form"},
|
||||
} {
|
||||
got, err := Parse(c.key)
|
||||
if c.why != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.why) || !strings.Contains(err.Error(), FormAgain) {
|
||||
t.Errorf("%s: read %+v (%v); want it refused, %s, naming the forms", c.key, got, err, c.why)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || got != c.want {
|
||||
t.Errorf("%s: read %+v (%v); want %+v", c.key, got, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -84,7 +84,14 @@ func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, erro
|
||||
if ask.Build != nil {
|
||||
job := *ask.Build
|
||||
job.Fetch = withPackages(job.Fetch, ask.Packages)
|
||||
return steps.build(ctx, job, ask.Version)
|
||||
out, err := steps.build(ctx, job, ask.Version)
|
||||
var dec *Declining
|
||||
if err != nil && !sdkstep.IsPassing(err) && !errors.As(err, &dec) && ctx.Err() == nil {
|
||||
// A failed build keeps its error text, as the build module bounded it, as the ask's output object, header and
|
||||
// text as a test step's, so the verb output answers why after the fact (design 47 H2 H-A-482 and its addendum (a)).
|
||||
steps.keepRunOutput(ask.ID, err.Error(), nil)
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
if ask.Step != nil || ask.Install != nil {
|
||||
return steps.runStep(ctx, ask)
|
||||
|
||||
@@ -117,6 +117,23 @@ imports {"module":"modules/m","workspace":"/w/1"}`
|
||||
if !slices.Equal(answer.Imports, []string{"git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/health"}) {
|
||||
t.Errorf("the answer's imports are %v", answer.Imports)
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-482 addendum (a): a failed build keeps its error text, as the build module bounded it, as the ask's
|
||||
// output object, which the verb output answers.
|
||||
outputs := fakeOutputs{m: map[string][]byte{}}
|
||||
steps.Outputs, steps.Node = outputs, "ace"
|
||||
steps.Build = func(context.Context, map[string]any) (any, error) {
|
||||
return nil, errors.New("esbuild for code: status 1 | ✘ [ERROR] Could not resolve \"./missing\" … 812 bytes left out … main.js:1:1)")
|
||||
}
|
||||
failed := ask(sdkstep.BuildJob{Module: "m", Path: "modules/m", Fetch: &sdkstep.Fetch{Repository: "novox/x", Commit: runCommit},
|
||||
Artifacts: []sdkstep.AskArtifact{{Artifact: "bundle"}}})
|
||||
if _, err := RunAsk(context.Background(), steps, failed); err == nil {
|
||||
t.Fatal("a failed build answered no error")
|
||||
}
|
||||
got2, err := readOutput(outputs, "ace", failed.ID, nil)
|
||||
if err != nil || !strings.Contains(got2.Output, `bundle: esbuild for code: status 1 | ✘ [ERROR] Could not resolve "./missing"`) || got2.Size != len(got2.Output) {
|
||||
t.Errorf("the failed build's output = %+v, %v; want its error text kept", got2, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build ask for another toolchain is declined (ADR 0584 clause 2), so the node sits out and the asker puts it again.
|
||||
|
||||
@@ -125,7 +125,8 @@ func EndedHere(node string, keys func() ([]string, error), get func(key string)
|
||||
// readOutput answers the verb output for ask (design 47 H2 §791, H-A-412): from the object `<node>.<ask>`, the text's last
|
||||
// OutputTail bytes, its size and failing tests from the header; with no object, no longer kept when this node's steps key
|
||||
// shows the ask ended here more than OutputsMaxAge ago, and refused otherwise, naming why (an ask never run here, or one
|
||||
// whose output this node did not keep).
|
||||
// that kept no output). An empty object is no object (H-A-482): the runtime's object read answers an absent object as
|
||||
// empty data without an error, which is not a header fault.
|
||||
func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.Time, bool, error)) (sdkstep.OutputAnswer, error) {
|
||||
if o == nil {
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("this module keeps no %s store", OutputsStore)
|
||||
@@ -135,11 +136,14 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.
|
||||
return sdkstep.OutputAnswer{}, err
|
||||
}
|
||||
got, gerr := o.Get(name)
|
||||
if gerr == nil {
|
||||
if gerr == nil && len(got.Data) > 0 {
|
||||
return outputOf(ask, got.Data)
|
||||
}
|
||||
if gerr == nil {
|
||||
gerr = fmt.Errorf("the object %s in the %s store is empty", name, OutputsStore)
|
||||
}
|
||||
if ended == nil {
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the output %s in the %s store: %w", name, OutputsStore, gerr)
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s kept no output: %w", ask, gerr)
|
||||
}
|
||||
at, here, err := ended(ask)
|
||||
switch {
|
||||
@@ -150,7 +154,7 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.
|
||||
case !at.IsZero() && now().Sub(at) > OutputsMaxAge:
|
||||
return sdkstep.OutputAnswer{Ask: ask, NoLongerKept: true}, nil
|
||||
default:
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and its output is not in the %s store: %w", ask, node, OutputsStore, gerr)
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and the ask kept no output: %w", ask, node, gerr)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -64,7 +64,9 @@ func TestOutputsKeptAreRead(t *testing.T) {
|
||||
return raw
|
||||
}
|
||||
steps := map[string]json.RawMessage{"mesh-delivery-pipeline.ace.a2": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a3": step(at.Add(-time.Hour)),
|
||||
"mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour))}
|
||||
"mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a6": step(at.Add(-time.Hour))}
|
||||
// H-A-482: the runtime's object read answers an absent object as empty data, no error; that is no output, not a header fault.
|
||||
out.m["ace.a6"] = []byte{}
|
||||
keys := func() ([]string, error) {
|
||||
return slices.Collect(func(y func(string) bool) {
|
||||
for k := range steps {
|
||||
@@ -82,7 +84,8 @@ func TestOutputsKeptAreRead(t *testing.T) {
|
||||
}{
|
||||
{ask: "a1", want: sdkstep.OutputAnswer{Ask: "a1", Output: long[len(long)-OutputTail:], Size: len(long), Failing: failing}},
|
||||
{ask: "a2", want: sdkstep.OutputAnswer{Ask: "a2", NoLongerKept: true}},
|
||||
{ask: "a3", fail: "ran on ace, and its output is not in the outputs store"},
|
||||
{ask: "a3", fail: "ran on ace, and the ask kept no output"},
|
||||
{ask: "a6", fail: "ran on ace, and the ask kept no output: the object ace.a6 in the outputs store is empty"},
|
||||
{ask: "a4", fail: "never ran on ace"},
|
||||
{ask: "a5", fail: "never ran on ace"},
|
||||
{fail: "asked for no ask"},
|
||||
|
||||
@@ -267,8 +267,9 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
|
||||
return nil
|
||||
}
|
||||
|
||||
// keepRunOutput keeps a test step's level text as the run's output `<node>.<ask id>` in the outputs objects, with the
|
||||
// header of the failing tests and their words from the level's results (design 47 H2 §791, H-A-412 (1)). A holder that
|
||||
// keepRunOutput keeps a test step's level text, or a failed build's error text (H-A-482), as the run's output `<node>.<ask
|
||||
// id>` in the outputs objects, with the header of the failing tests and their words from the level's results (design 47
|
||||
// H2 §791, H-A-412 (1)). A holder that
|
||||
// keeps no outputs, or a write that fails, is said in the journal: the step's end stands on the results.
|
||||
func (s AskSteps) keepRunOutput(ask, text string, failed []sdkstep.Failure) {
|
||||
var failing []sdkstep.FailingTest
|
||||
|
||||
@@ -5,13 +5,15 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The part's own membership names the stream running among those it hears, and hearRunning binds the part's durable on it:
|
||||
// The part's own membership names the stream running among those it hears, and hearRunning binds the part's durable on it
|
||||
// (and EVENTS, which the hub reads, H-A-472 seventh addendum):
|
||||
// the loop that opens the other streams leaves it alone, so no second reader takes messages of that one durable (design 47 G
|
||||
// §556, A-19; the whole-part integration test lost the running event to the second reader at random).
|
||||
func TestTheLoopOfOwnHearsLeavesTheStreamsADedicatedHearingOwns(t *testing.T) {
|
||||
hears := []membership.HeardIssued{{Stream: nodeengine.StreamRunning, Consumer: "a_mesh-grants"}, {Stream: "mod-c_feed", Consumer: "a_mesh-grants"}, {Stream: "opened_one"}}
|
||||
hears := []membership.HeardIssued{{Stream: nodeengine.StreamRunning, Consumer: "a_mesh-grants"}, {Stream: "mod-c_feed", Consumer: "a_mesh-grants"}, {Stream: "opened_one"}, {Stream: process.EventsStream, Consumer: "a_mesh-grants"}}
|
||||
got := hearsToOpen(hears, map[string]bool{"opened_one": true})
|
||||
if len(got) != 1 || got[0].Stream != "mod-c_feed" {
|
||||
t.Errorf("the streams opened are %+v, want mod-c_feed alone", got)
|
||||
|
||||
@@ -110,7 +110,10 @@ func TestTheGrantsPartWhole(t *testing.T) {
|
||||
}
|
||||
manifest("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},`+
|
||||
`"emits":["created","done","dropped","recreated"],"streams":[{"name":"broker-events","events":["created","dropped","recreated"],"max-age-days":30,"max-mib":256}]}`)
|
||||
manifest("mod-c", `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["nats.done"]}`)
|
||||
// mod-c names a program of its own, so its program's consumer a_mod-c is composed beside the tool runner's twin (design 47
|
||||
// H2, H-A-472 second addendum).
|
||||
manifest("mod-c", `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["nats.done"],`+
|
||||
`"resources":[{"id":"program","type":"process","run":["./mod-c"]}]}`)
|
||||
// The part itself, placed as on the mesh, so its own consumer on EVENTS hears the seat mesh-broker's events (DLV-G1).
|
||||
own, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
|
||||
@@ -188,6 +188,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
f.Part.RevisionsPerBranch, f.Part.Followed = perBranch, branches.Followed
|
||||
// The setting twin-current-hours, read at the start: the bound of a current twin for the orphaned durables (design 47 H2,
|
||||
// H-A-472 third addendum).
|
||||
if f.Part.TwinCurrentHours, err = grants.TwinCurrentHoursSetting(strings.TrimSpace(getenv(grants.EnvTwinCurrentHours))); err != nil {
|
||||
return err
|
||||
}
|
||||
f.Watch = fv
|
||||
f.Part.ReadingRunning(nodeengine.StreamRunning)
|
||||
// The catalog-revision event names each branch's newest revision, so grants-for asks the seat mesh-catalog's revision only
|
||||
@@ -308,11 +313,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// hearsToOpen are the streams of the part's own membership no hearing has opened yet. A stream another hearing of the part
|
||||
// binds its durable on (running: hearRunning, which reads it whole at the start before it follows) is left to that hearing:
|
||||
// two readers of one durable each take the messages the other does not, and the event a reader of its own never hears is lost
|
||||
// to it (the integration test of the whole part met it as a state running that stayed empty, 2026-11-12).
|
||||
// to it (the integration test of the whole part met it as a state running that stayed empty, 2026-11-12). So is EVENTS, which
|
||||
// the membership names like any stream since it carries the consumer's spec (design 47 H2, H-A-472 seventh addendum): the
|
||||
// hub reads the part's durable there.
|
||||
func hearsToOpen(hears []membership.HeardIssued, opened map[string]bool) []membership.HeardIssued {
|
||||
var out []membership.HeardIssued
|
||||
for _, h := range hears {
|
||||
if !opened[h.Stream] && h.Stream != nodeengine.StreamRunning {
|
||||
if !opened[h.Stream] && h.Stream != nodeengine.StreamRunning && h.Stream != process.EventsStream {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,11 +314,14 @@ func recordHandActAs(cause string) func(what, why string) error {
|
||||
func serveTools() error {
|
||||
tv := &toolViews{}
|
||||
gate := newSeatGate()
|
||||
p := &grants.Part{Read: runtimeReader{views: tv}, Call: runtimeBus{}, Store: grants.BigValues(runtimeStore{}, nil), Emit: runtimeBus{}, SeatList: gate}
|
||||
// Its journal is its standard error, which the tool runner keeps: grants-for says each stage's time there (H-A-478).
|
||||
logf := func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-grants: "+f+"\n", a...) }
|
||||
p := &grants.Part{Read: runtimeReader{views: tv}, Call: runtimeBus{}, Store: grants.BigValues(runtimeStore{}, nil), Emit: runtimeBus{}, SeatList: gate,
|
||||
Logf: logf}
|
||||
// The reader asks through the runtime, which serves once Serve has its channel.
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-grants: "+f+"\n", a...) })
|
||||
gate.Follow(seatlist.StdioBus(), nil, logf)
|
||||
}()
|
||||
tools := append(toolsOf(p), gate.Tools()...)
|
||||
for i := range tools {
|
||||
|
||||
@@ -547,6 +547,10 @@ func (b *fakeBus) setAssignments(pl meshmoduleassigner.Assignments) {
|
||||
maps.Copy(b.states[grants.ReadAssignments], values)
|
||||
}
|
||||
|
||||
// programResource names a program of the module's own, a process resource with no bus of its own: its consumer
|
||||
// <node>_<module> is composed beside the tool runner's twin (design 47 H2, H-A-472 second addendum).
|
||||
const programResource = `,"resources":[{"id":"program","type":"process","run":["./program"]}]`
|
||||
|
||||
const (
|
||||
natsManifest = `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"}}`
|
||||
xManifest = `{"module":"mod-x","version":"1","own-secrets":{"broker":"/var/lib/x/broker"},"state":[{"name":"kept","history":3}],` +
|
||||
@@ -973,7 +977,8 @@ func TestGrants(t *testing.T) {
|
||||
if again := p.Compose(); again.Withheld == nil || len(again.Withheld.Written) != 0 || again.Withheld.Withheld != 1 {
|
||||
t.Errorf("a second pass wrote %+v, want nothing (on change only)", again.Withheld)
|
||||
}
|
||||
b.manifestOf("reader", `{"module":"reader","version":"1","runs-as":"reader","resources":[{"id":"account","type":"user","name":"reader","shell":"/usr/bin/nologin","home":"/var/lib/reader"}],`+
|
||||
b.manifestOf("reader", `{"module":"reader","version":"1","runs-as":"reader","resources":[{"id":"account","type":"user","name":"reader","shell":"/usr/bin/nologin","home":"/var/lib/reader"},`+
|
||||
`{"id":"program","type":"process","run":["./reader"]}],`+
|
||||
`"own-secrets":{"broker":"/var/lib/r/broker"},"secrets-owner":"reader","consumes":["mesh-mail.mail.box.message.added"]}`)
|
||||
if heard := p.Compose(); heard.Withheld == nil || !slices.Equal(heard.Withheld.Deleted, []string{key}) || b.own[grants.StateWithheld][key] != nil {
|
||||
t.Errorf("once reader runs as its own account the pass did %+v (errors %v), want %s deleted", heard.Withheld, heard.Errors, key)
|
||||
@@ -2244,7 +2249,7 @@ func TestGrants(t *testing.T) {
|
||||
// A seat's event the seat list says is kept in a stream of its own (design 47): its stream is made once, and the module that
|
||||
// consumes it has its durable on that stream, none on the shared one.
|
||||
kb := fixture()
|
||||
kb.manifestOf("mod-y", `{"module":"mod-y","version":"1","own-secrets":{"broker":"/var/lib/y/broker"},"consumes":["mesh-conditions.condition-raised"]}`)
|
||||
kb.manifestOf("mod-y", `{"module":"mod-y","version":"1","own-secrets":{"broker":"/var/lib/y/broker"},"consumes":["mesh-conditions.condition-raised"]`+programResource+`}`)
|
||||
kb.loaded = shadowOf(t, kb)
|
||||
kp := partOn(kb)
|
||||
kp.Create, kp.Consumers = kb, kb
|
||||
@@ -2401,7 +2406,7 @@ func TestGrants(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("account consumers: a held consumer gains the filters of a new event, never loses one", func(t *testing.T) {
|
||||
const cManifest = `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["mod-y.done","mod-y.more"]}`
|
||||
const cManifest = `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["mod-y.done","mod-y.more"]` + programResource + `}`
|
||||
b := fixture()
|
||||
b.manifestOf("mod-c", cManifest)
|
||||
b.assign("b", "mod-c")
|
||||
@@ -3152,6 +3157,24 @@ func TestGrants(t *testing.T) {
|
||||
if got := slices.Sorted(maps.Keys(b.own[grants.StateMemberships])); !slices.Equal(got, []string{"a.mod-x", "a.nats", "b.mod-y"}) {
|
||||
t.Errorf("memberships %v", got)
|
||||
}
|
||||
// Design 47 H2, H-A-472 ninth addendum: a module no longer assigned on its node has its membership withdrawn, so the
|
||||
// tool runner there removes its twins; a pass whose read failed withdraws nothing, and a bindings key is no membership.
|
||||
b.mu.Lock()
|
||||
b.own[grants.StateMemberships][grants.BoundKey("mod-y")] = []byte(`{}`)
|
||||
b.unassign("b", "mod-y")
|
||||
b.refused[grants.ReadHave] = true
|
||||
b.mu.Unlock()
|
||||
p.Compose()
|
||||
if _, _, found, _ := b.ownGet(grants.StateMemberships, "b.mod-y"); !found {
|
||||
t.Error("a pass whose read failed withdrew b.mod-y")
|
||||
}
|
||||
b.mu.Lock()
|
||||
delete(b.refused, grants.ReadHave)
|
||||
b.mu.Unlock()
|
||||
p.Compose()
|
||||
if got := slices.Sorted(maps.Keys(b.own[grants.StateMemberships])); !slices.Equal(got, []string{"a.mod-x", "a.nats", grants.BoundKey("mod-y")}) {
|
||||
t.Errorf("memberships once mod-y left b: %v", got)
|
||||
}
|
||||
for _, r := range grants.Reads {
|
||||
if strings.HasPrefix(r, "mesh-controller.") {
|
||||
t.Errorf("the part reads %s", r)
|
||||
@@ -3305,7 +3328,7 @@ func TestGrants(t *testing.T) {
|
||||
// The part's own consumer on the bus module's stream, and every module's on a module stream it hears, is created by
|
||||
// the part since the controller ended (ADR 0363 §5, ADR 0590): here mod-y's on mod-x's THINGS.
|
||||
heard := fixture()
|
||||
heard.manifestOf("mod-y", strings.Replace(yManifest, `"reads"`, `"consumes":["mod-x.thing.added"],"reads"`, 1))
|
||||
heard.manifestOf("mod-y", strings.Replace(yManifest, `"reads"`, `"consumes":["mod-x.thing.added"],"resources":[{"id":"program","type":"process","run":["./y"]}],"reads"`, 1))
|
||||
hp := partOn(heard)
|
||||
hp.Consumers = heard
|
||||
hp.Compose()
|
||||
@@ -3769,14 +3792,17 @@ func TestGrants(t *testing.T) {
|
||||
p := partOn(b)
|
||||
p.Consumers = b
|
||||
p.Compose()
|
||||
for _, c := range []string{"INDEXER_VIEWS.a_node-tools", "ASSIGNER_VIEWS.a_node-tools"} {
|
||||
if !slices.Contains(b.consumerCreates, c) {
|
||||
t.Errorf("the tool runner's consumer %s was not made: %v", c, b.consumerCreates)
|
||||
// The runner names no process of its own, so its consumers are its twins alone (H-A-472 second addendum), which the
|
||||
// runner creates itself from its membership's spec, granted their create by name (fifth to seventh addenda).
|
||||
for _, c := range []string{"INDEXER_VIEWS.a_node-tools_tools", "ASSIGNER_VIEWS.a_node-tools_tools"} {
|
||||
if slices.Contains(b.consumerCreates, c) {
|
||||
t.Errorf("the part made the tool runner's twin %s: %v", c, b.consumerCreates)
|
||||
}
|
||||
}
|
||||
users := shadowOf(t, b)
|
||||
runner := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "a.node-tools" })]
|
||||
for _, s := range []string{"$JS.API.CONSUMER.MSG.NEXT.INDEXER_VIEWS.a_node-tools", "$KV.node-tools_findings.>"} {
|
||||
for _, s := range []string{"$JS.API.CONSUMER.MSG.NEXT.INDEXER_VIEWS.a_node-tools", "$KV.node-tools_findings.>",
|
||||
"$JS.API.CONSUMER.CREATE.INDEXER_VIEWS.a_node-tools_tools", "$JS.API.CONSUMER.DELETE.ASSIGNER_VIEWS.a_node-tools_tools"} {
|
||||
if !slices.ContainsFunc(runner.Publish, func(g string) bool { return buscompose.SubjectsOverlap(g, s) }) {
|
||||
t.Errorf("the tool runner may not publish %s: %v", s, runner.Publish)
|
||||
}
|
||||
@@ -4140,7 +4166,7 @@ func TestGrants(t *testing.T) {
|
||||
// configuration, are created where the bus lacks them with no lease key at all (the controller ended by its
|
||||
// unassignment, never by the lease write); one the bus holds is never changed.
|
||||
t.Run("account consumers: created where the bus lacks them, with no lease key", func(t *testing.T) {
|
||||
const cManifest = `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["mod-y.done"]}`
|
||||
const cManifest = `{"module":"mod-c","version":"1","own-secrets":{"broker":"/var/lib/c/broker"},"consumes":["mod-y.done"]` + programResource + `}`
|
||||
b := fixture()
|
||||
b.manifestOf("mod-c", cManifest)
|
||||
b.assign("b", "mod-c")
|
||||
@@ -4149,7 +4175,7 @@ func TestGrants(t *testing.T) {
|
||||
p.Consumers = b
|
||||
b.consumers = map[string][]map[string]any{"EVENTS": {{"name": "b_other", "filters": []string{"x"}, "max_deliver": 5}}}
|
||||
res := p.Compose()
|
||||
if res.Consumers == nil || !slices.Equal(res.Consumers.Created, []string{"EVENTS.b_mod-c", "EVENTS.b_mod-c_tools"}) {
|
||||
if res.Consumers == nil || !slices.Equal(res.Consumers.Created, []string{"EVENTS.b_mod-c"}) {
|
||||
t.Fatalf("with no lease key: %+v", res.Consumers)
|
||||
}
|
||||
var made map[string]any
|
||||
@@ -4162,17 +4188,19 @@ func TestGrants(t *testing.T) {
|
||||
t.Errorf("b_mod-c was made as %v, not as the controller makes it", made)
|
||||
}
|
||||
res = p.Compose()
|
||||
if len(res.Consumers.Created) != 0 || len(res.Consumers.Missing) != 0 || len(b.consumerCreates) != 2 {
|
||||
if len(res.Consumers.Created) != 0 || len(res.Consumers.Missing) != 0 || len(b.consumerCreates) != 1 {
|
||||
t.Errorf("a consumer the bus holds was asked again: %+v, %v", res.Consumers, b.consumerCreates)
|
||||
}
|
||||
// A held consumer that differs is never changed by a pass: it is raised as a condition naming it, and brought to
|
||||
// the composed configuration only by hand (grants_consumer_match), with no lease key too.
|
||||
var said, behind []oc.Condition
|
||||
var said, behind, orphaned []oc.Condition
|
||||
p.Say = func(c oc.Condition) error {
|
||||
switch c.Key {
|
||||
case grants.GrantMissingCondition, grants.RefusedCondition, grants.HeldModuleCondition, grants.FullCondition: // their own rows
|
||||
case grants.ConsumerBehindCondition:
|
||||
behind = append(behind, c)
|
||||
case grants.DurableOrphanedCondition:
|
||||
orphaned = append(orphaned, c)
|
||||
default:
|
||||
said = append(said, c)
|
||||
}
|
||||
@@ -4202,7 +4230,8 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
// 1126 (the self-check's bus-consumer probe ended): a consumer the part owns past the bound behind its stream's
|
||||
// head at two passes running raises mesh-grants.consumer-behind naming it; at the bound or under, it is cleared.
|
||||
made["pending"] = grants.BehindBound + 1
|
||||
// Read once, no pull now, far behind: behind (H-A-472 eleventh addendum), never an orphan.
|
||||
made["pending"], made["last_delivered"] = grants.BehindBound+1, "2026-10-20T21:00:00Z"
|
||||
if res = p.Compose(); len(res.Consumers.Behind) != 0 {
|
||||
t.Errorf("one look at a burst was said far behind: %+v", res.Consumers)
|
||||
}
|
||||
@@ -4222,6 +4251,127 @@ func TestGrants(t *testing.T) {
|
||||
if len(res.Consumers.Behind) != 0 || behind[len(behind)-1].State != occ.StateCleared {
|
||||
t.Errorf("a consumer back within the bound: %+v, said %+v", res.Consumers, behind)
|
||||
}
|
||||
// Design 47 H2, H-A-472 (2): a durable far behind with no pulls waiting, whose twin on the stream (b_mod-c_tools) has
|
||||
// pulls waiting and delivered within the day, is an orphan: left out of the behind check, named once in
|
||||
// mesh-grants.durable-orphaned, and cleared when it is gone; with its twin not current, it is behind again.
|
||||
// The twin is the tool runner's, made by it from the membership's spec (the fifth addendum), never by the part.
|
||||
twin := map[string]any{"name": "b_mod-c_tools", "filters": []string{"mesh.mod.mod-y.event.done"}, "max_deliver": 5}
|
||||
b.consumers["EVENTS"] = append(b.consumers["EVENTS"], twin)
|
||||
twin["waiting_pulls"], twin["last_delivered"] = 1, "2026-10-25T21:00:00.5Z"
|
||||
made["pending"] = grants.BehindBound + 1
|
||||
p.Compose()
|
||||
o := len(orphaned)
|
||||
if res = p.Compose(); len(res.Consumers.Behind) != 0 || len(orphaned) != o+1 || orphaned[o].State != occ.StateRaised ||
|
||||
!strings.Contains(orphaned[o].Explanation, "EVENTS.b_mod-c") || strings.Contains(orphaned[o].Explanation, "EVENTS.b_mod-c_tools") {
|
||||
t.Fatalf("an orphaned durable: %+v, behind said %+v, orphaned said %+v", res.Consumers, behind[len(behind)-1], orphaned)
|
||||
}
|
||||
if p.Compose(); len(orphaned) != o+1 {
|
||||
t.Fatalf("an orphan was said again: %+v", orphaned)
|
||||
}
|
||||
twin["last_delivered"] = "2026-10-23T21:00:00Z"
|
||||
if res = p.Compose(); len(res.Consumers.Behind) != 1 || len(orphaned) != o+2 || orphaned[o+1].State != occ.StateCleared {
|
||||
t.Fatalf("a durable whose twin delivered two days ago: %+v, orphaned said %+v", res.Consumers, orphaned)
|
||||
}
|
||||
// The third addendum (2): the bound is the setting twin-current-hours; at 72 hours, a twin that delivered two days
|
||||
// ago is current, so the durable is an orphan again; the setting's file gives its hours, its default when missing.
|
||||
p.TwinCurrentHours = 72
|
||||
if res = p.Compose(); len(res.Consumers.Behind) != 0 || len(orphaned) != o+3 || orphaned[o+2].State != occ.StateRaised {
|
||||
t.Fatalf("with twin-current-hours 72, a durable whose twin delivered two days ago: %+v, orphaned said %+v", res.Consumers, orphaned)
|
||||
}
|
||||
p.TwinCurrentHours = 0
|
||||
if res = p.Compose(); len(res.Consumers.Behind) != 1 || len(orphaned) != o+4 || orphaned[o+3].State != occ.StateCleared {
|
||||
t.Fatalf("back at the default bound: %+v, orphaned said %+v", res.Consumers, orphaned)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
text string
|
||||
hours int
|
||||
fails bool
|
||||
}{{"72\n", 72, false}, {"", grants.DefaultTwinCurrentHours, false}, {"0", 0, true}, {"x", 0, true}} {
|
||||
path := filepath.Join(t.TempDir(), "twin-current-hours")
|
||||
if err := os.WriteFile(path, []byte(row.text), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := grants.TwinCurrentHoursSetting(path); n != row.hours || (err != nil) != row.fails {
|
||||
t.Errorf("twin-current-hours %q: %d, %v", row.text, n, err)
|
||||
}
|
||||
}
|
||||
if n, err := grants.TwinCurrentHoursSetting(filepath.Join(t.TempDir(), "none")); n != 24 || err != nil {
|
||||
t.Errorf("with no file twin-current-hours is %d, %v; want 24", n, err)
|
||||
}
|
||||
made["pending"] = grants.BehindBound
|
||||
p.Compose()
|
||||
// H-A-472 second and fifth addenda: a module with an account of its own but no process resource with no bus of its own
|
||||
// has no program reading b_<module>, and the tool runner makes its own twin, so the part makes neither.
|
||||
b.manifestOf("mod-t", `{"module":"mod-t","version":"1","own-secrets":{"broker":"/var/lib/t/broker"},"consumes":["mod-y.done"]}`)
|
||||
b.assign("b", "mod-t")
|
||||
if res = p.Compose(); slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t_tools") || slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t") {
|
||||
t.Errorf("a module with no program of its own: created %v", res.Consumers.Created)
|
||||
}
|
||||
// The tenth and eleventh addenda: every held durable no pass composes is still judged, both twins of a module.
|
||||
named := func(list []string, key string) bool {
|
||||
return slices.ContainsFunc(list, func(s string) bool { return s == key || strings.HasPrefix(s, key+":") })
|
||||
}
|
||||
lastOrphans := func() string {
|
||||
if len(orphaned) == 0 || orphaned[len(orphaned)-1].State != occ.StateRaised {
|
||||
return ""
|
||||
}
|
||||
return orphaned[len(orphaned)-1].Explanation
|
||||
}
|
||||
// b_mod-c_tools, no pull waiting while the program's b_mod-c is current: an orphan by (2).
|
||||
twin["waiting_pulls"], twin["pending"] = 0, grants.BehindBound+1
|
||||
made["waiting_pulls"], made["last_delivered"], made["pending"] = 1, "2026-10-25T21:00:00.5Z", 0
|
||||
// b_mod-t_tools, never read (no pull, no delivery ever), mod-t having no program: an orphan, never behind.
|
||||
tTools := map[string]any{"name": "b_mod-t_tools", "filters": []string{"mesh.mod.mod-y.event.done"}, "max_deliver": 5,
|
||||
"pending": grants.BehindBound + 1}
|
||||
b.consumers["EVENTS"] = append(b.consumers["EVENTS"], tTools)
|
||||
p.Compose()
|
||||
res = p.Compose()
|
||||
if named(res.Consumers.Behind, "EVENTS.b_mod-t_tools") || named(res.Consumers.Behind, "EVENTS.b_mod-c_tools") ||
|
||||
!strings.Contains(lastOrphans(), "EVENTS.b_mod-c_tools") || !strings.Contains(lastOrphans(), "EVENTS.b_mod-t_tools") {
|
||||
t.Errorf("the held twins: behind %v, orphaned said %q", res.Consumers.Behind, lastOrphans())
|
||||
}
|
||||
// Read once, no pull now, far behind, with no current twin: behind.
|
||||
tTools["last_delivered"] = "2026-10-20T21:00:00Z"
|
||||
p.Compose()
|
||||
if res = p.Compose(); !named(res.Consumers.Behind, "EVENTS.b_mod-t_tools") || strings.Contains(lastOrphans(), "EVENTS.b_mod-t_tools") {
|
||||
t.Errorf("a twin read once and lagging: behind %v, orphaned said %q", res.Consumers.Behind, lastOrphans())
|
||||
}
|
||||
// The plain b_mod-t, which no pass composes (mod-t's program left), far behind with no pull beside a current
|
||||
// b_mod-t_tools: an orphan by (2).
|
||||
tTools["waiting_pulls"], tTools["last_delivered"], tTools["pending"] = 1, "2026-10-25T21:00:00.5Z", 0
|
||||
b.consumers["EVENTS"] = append(b.consumers["EVENTS"], map[string]any{"name": "b_mod-t", "filters": []string{"mesh.mod.mod-y.event.done"},
|
||||
"max_deliver": 5, "pending": grants.BehindBound + 1, "last_delivered": "2026-10-20T21:00:00Z"})
|
||||
p.Compose()
|
||||
if res = p.Compose(); named(res.Consumers.Behind, "EVENTS.b_mod-t") || !strings.Contains(lastOrphans(), "EVENTS.b_mod-t,") &&
|
||||
!strings.HasSuffix(lastOrphans(), "EVENTS.b_mod-t") {
|
||||
t.Errorf("the plain durable of a module whose program left: behind %v, orphaned said %q", res.Consumers.Behind, lastOrphans())
|
||||
}
|
||||
// A stream only twins are held on (no pass composes a consumer there): mod-s keeps the stream feed, which the
|
||||
// program-less mod-u hears; its never-read twin there is judged and named.
|
||||
b.manifestOf("mod-s", `{"module":"mod-s","version":"1","emits":["fed"],"streams":[{"name":"feed","events":["fed"],"max-age-days":2,"max-mib":5}]}`)
|
||||
b.manifestOf("mod-u", `{"module":"mod-u","version":"1","own-secrets":{"broker":"/var/lib/u/broker"},"consumes":["mod-s.fed"]}`)
|
||||
b.assign("b", "mod-s")
|
||||
b.assign("b", "mod-u")
|
||||
p.Compose()
|
||||
var feed string
|
||||
raw, _, _, _ := b.ownGet(grants.StateMemberships, "b.mod-u")
|
||||
if bare, err := membership.BareGrantsMembership(raw); err == nil {
|
||||
var m membership.Membership
|
||||
_ = json.Unmarshal(bare, &m)
|
||||
for _, h := range m.Hears {
|
||||
if h.Stream != "EVENTS" {
|
||||
feed = h.Stream
|
||||
}
|
||||
}
|
||||
}
|
||||
if feed == "" {
|
||||
t.Fatalf("mod-u's membership hears no stream of mod-s: %s", raw)
|
||||
}
|
||||
b.consumers[feed] = []map[string]any{{"name": "b_mod-u_tools", "filters": []string{"x"}, "max_deliver": 5, "pending": grants.BehindBound + 1}}
|
||||
p.Compose()
|
||||
if res = p.Compose(); !strings.Contains(lastOrphans(), feed+".b_mod-u_tools") {
|
||||
t.Errorf("a twin on a stream no pass composes a consumer on: orphaned said %q (unread %v)", lastOrphans(), res.Consumers.Unread)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/AC13 (novox/hq ADR 0561 §7): after take-over a list in which more than one loaded user's hash changes, with no
|
||||
@@ -5275,6 +5425,52 @@ func TestGrantsFor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// grants-for composes each revision's grants once and keeps them with the revision (design 47 H2 H-A-478 and its addendum):
|
||||
// a second ask for the same revisions composes nothing again; an ask of another candidate against the same newest composes
|
||||
// the candidate alone; each stage's time is in the journal.
|
||||
func TestGrantsForComposesARevisionOnce(t *testing.T) {
|
||||
candidateX := strings.Replace(xManifest, `"state"`, `"emits":["hello"],"state"`, 1)
|
||||
b := fixture()
|
||||
b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": candidateX, "mod-y": xManifest}}
|
||||
b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8}
|
||||
p := partOn(b)
|
||||
var journal []string
|
||||
p.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
|
||||
ask := func(in meshgrants.GrantsForInput) (meshgrants.GrantsFor, string) {
|
||||
t.Helper()
|
||||
journal = nil
|
||||
got, err := p.GrantsFor(in)
|
||||
if err != nil || got.Error != "" {
|
||||
t.Fatalf("grants-for: %+v, %v", got, err)
|
||||
}
|
||||
if len(journal) != 1 || !strings.HasPrefix(journal[0], "grants-for main ") || !strings.Contains(journal[0], "the inputs ") {
|
||||
t.Fatalf("grants-for said %q, want one line of its stages' times", journal)
|
||||
}
|
||||
return got, journal[0]
|
||||
}
|
||||
first, said := ask(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
if len(first.Modules) != 1 || first.Modules[0].Module != "mod-x" || strings.Count(said, "(composed)") != 2 {
|
||||
t.Errorf("the first ask answered %+v and said %q, want mod-x and both revisions' grants composed", first, said)
|
||||
}
|
||||
again, said := ask(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
if strings.Contains(said, "composed") || strings.Count(said, "(kept)") != 2 || !reflect.DeepEqual(first, again) {
|
||||
t.Errorf("a second ask for the same revisions said %q and answered %+v, want both kept and %+v", said, again, first)
|
||||
}
|
||||
// Inputs that differ compose both again (H-A-478 second addendum); the same inputs then keep them again.
|
||||
b.put(grants.ReadResolved, "a.mod-x.greeting", map[string]any{"node": "a", "module": "mod-x", "key": "greeting", "value": "hello"})
|
||||
if _, said = ask(meshgrants.GrantsForInput{Branch: "main", Source: "7"}); strings.Count(said, "(composed)") != 2 {
|
||||
t.Errorf("an ask whose inputs differ said %q, want both revisions' grants composed again", said)
|
||||
}
|
||||
if _, said = ask(meshgrants.GrantsForInput{Branch: "main", Source: "7"}); strings.Count(said, "(kept)") != 2 {
|
||||
t.Errorf("an ask with the same inputs again said %q, want both kept", said)
|
||||
}
|
||||
// A run's candidate is another revision: composed once, the newest's grants kept.
|
||||
_, said = ask(meshgrants.GrantsForInput{Branch: "main", Run: "r1"})
|
||||
if !strings.Contains(said, "the newest's grants (kept)") || !strings.Contains(said, "the candidate's grants (composed)") {
|
||||
t.Errorf("an ask of another candidate said %q, want the newest's grants kept and the candidate's composed", said)
|
||||
}
|
||||
}
|
||||
|
||||
// grants-for keeps what H-A-195 says of the catalog: so many revisions to a branch as the setting
|
||||
// grants-for-revisions-per-branch says, a run's candidate counted within its branch's; only the followed branches', freed when
|
||||
// one leaves them; and an event below the newest held is the numbering started again, which drops what is kept.
|
||||
@@ -5305,8 +5501,11 @@ func TestGrantsForKeepsWhatItMay(t *testing.T) {
|
||||
perBranch int
|
||||
then func(t *testing.T, b *fakeBus, p *grants.Part, followed *[]string)
|
||||
pages int
|
||||
// composed is how many of the two revisions' grants the grants-for after it composes again: a revision no longer
|
||||
// kept has no grants kept either (H-A-478).
|
||||
composed int
|
||||
}{
|
||||
{name: "the setting bounds a branch's revisions: with 2, the candidate is dropped once two newer are kept", pages: 2,
|
||||
{name: "the setting bounds a branch's revisions: with 2, the candidate is dropped once two newer are kept", pages: 2, composed: 1,
|
||||
then: func(t *testing.T, b *fakeBus, p *grants.Part, _ *[]string) {
|
||||
p.RevisionsPerBranch = setting(t, "2\n")
|
||||
hear(t, p, 10, 0)
|
||||
@@ -5314,14 +5513,14 @@ func TestGrantsForKeepsWhatItMay(t *testing.T) {
|
||||
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
}},
|
||||
{name: "a run's candidate counts within its branch's bound: with 1, the candidate and the newest each drop the other", run: true,
|
||||
perBranch: 1, pages: 4, then: func(*testing.T, *fakeBus, *grants.Part, *[]string) {}},
|
||||
{name: "a branch that leaves the watched branches frees its revisions", pages: 4,
|
||||
perBranch: 1, pages: 4, composed: 2, then: func(*testing.T, *fakeBus, *grants.Part, *[]string) {}},
|
||||
{name: "a branch that leaves the watched branches frees its revisions", pages: 4, composed: 2,
|
||||
then: func(t *testing.T, _ *fakeBus, p *grants.Part, followed *[]string) {
|
||||
*followed = nil
|
||||
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
*followed = []string{"main"}
|
||||
}},
|
||||
{name: "an event below the newest held drops what is kept of the branch", pages: 4,
|
||||
{name: "an event below the newest held drops what is kept of the branch", pages: 4, composed: 2,
|
||||
then: func(t *testing.T, _ *fakeBus, p *grants.Part, _ *[]string) { hear(t, p, 3, 0) }},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
@@ -5344,10 +5543,15 @@ func TestGrantsForKeepsWhatItMay(t *testing.T) {
|
||||
}
|
||||
row.then(t, b, p, &followed)
|
||||
before := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision }))
|
||||
var said []string
|
||||
p.Logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
_, _ = p.GrantsFor(in)
|
||||
if got := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision })) - before; got != row.pages {
|
||||
t.Errorf("grants-for read %d revision pages, want %d", got, row.pages)
|
||||
}
|
||||
if len(said) != 1 || strings.Count(said[0], "(composed)") != row.composed {
|
||||
t.Errorf("grants-for said %q, want %d of the revisions' grants composed again", said, row.composed)
|
||||
}
|
||||
})
|
||||
}
|
||||
if n, err := grants.RevisionsPerBranchSetting(filepath.Join(t.TempDir(), "none")); n != grants.DefaultRevisionsPerBranch || err != nil {
|
||||
|
||||
@@ -618,7 +618,8 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
|
||||
}
|
||||
// The grants part is granted the create of each account's consumer by its stream and name, and no other.
|
||||
users, err := Users(Records{Nodes: []string{"a"}, Assigned: map[string][]Declared{"a": {
|
||||
{Module: GrantsModule}, {Module: "m", Consumes: []string{"x.happened"}}, {Module: "quiet"}}}})
|
||||
{Module: GrantsModule}, {Module: "m", Consumes: []string{"x.happened"}, HasProgram: true}, {Module: "quiet"},
|
||||
{Module: "tooled", Consumes: []string{"x.happened"}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -639,6 +640,13 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
|
||||
MayPublish(perms, "$JS.API.CONSUMER.DELETE.EVENTS.a_m") || MayPublish(perms, "$JS.API.CONSUMER.CREATE.EVENTS.a_m.mesh.mod.x.event.happened") {
|
||||
t.Errorf("the grants part creates consumers as %v", u.CreatesConsumers)
|
||||
}
|
||||
// Design 47 H2, H-A-472 second and fifth addenda: the program's consumer only for a module with a program of its own;
|
||||
// the tool runner's twin never, which the runner creates itself from the membership's spec.
|
||||
for durable, want := range map[string]bool{"a_m": true, "a_m_tools": false, "a_tooled": false, "a_tooled_tools": false} {
|
||||
if MayPublish(perms, "$JS.API.CONSUMER.CREATE.EVENTS."+durable) != want {
|
||||
t.Errorf("the grants part creates EVENTS.%s: %v, want %v (%v)", durable, !want, want, u.CreatesConsumers)
|
||||
}
|
||||
}
|
||||
}
|
||||
// 1952 S5 (novox/hq ADR 0564 clauses 2, 5, 8; Limits 2, 4, 8): the health holders' credentials.
|
||||
healthChecks(t)
|
||||
|
||||
@@ -163,6 +163,14 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
||||
}, true
|
||||
}
|
||||
|
||||
// carriedPrincipal is a module the tool runner carries as the consumers it reads are composed from: the one form ConsumersOf
|
||||
// composes a carried module's consumers and HeardIssuedFor their spec from, so the two cannot differ.
|
||||
func carriedPrincipal(node string, d Declared) Principal {
|
||||
return Principal{Kind: KindModule, Node: node, Module: d.Module, HasProgram: d.HasProgram, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||
Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues}
|
||||
}
|
||||
|
||||
// ModuleConsumer is one module's durable consumer, with the module and node it is for.
|
||||
type ModuleConsumer struct {
|
||||
Node, Module string
|
||||
@@ -191,8 +199,9 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
}
|
||||
seen[key] = true
|
||||
if needed {
|
||||
// Each with the tool runner's twin beside it (A-397).
|
||||
for _, tc := range withToolsTwin(c) {
|
||||
// The program's only, for a module with a program of its own (H-A-472 second addendum); the tool runner's twin is
|
||||
// the runner's to make from the membership's spec (H-A-472 fifth to seventh addenda).
|
||||
for _, tc := range twinsOf(c, p.HasProgram) {
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: tc})
|
||||
}
|
||||
// And one on each module stream it hears (novox/hq ADR 0363 §5).
|
||||
@@ -200,7 +209,7 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
if len(h.Filters) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, tc := range withToolsTwin(heardConsumer(p, h)) {
|
||||
for _, tc := range twinsOf(heardConsumer(p, h), p.HasProgram) {
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: tc})
|
||||
}
|
||||
}
|
||||
@@ -243,9 +252,7 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
// 0436 §1, §2, §6), the same reason its ordinary consumer is (ADR 0198): mesh-mail has no
|
||||
// account of its own (NoAccount) and is carried here, but its process does, and the consumer
|
||||
// it reads through must exist and survive PruneHeard whether or not the module itself speaks.
|
||||
add(Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||
Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues})
|
||||
add(carriedPrincipal(p.Node, d))
|
||||
}
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
@@ -254,28 +255,56 @@ func heardConsumer(p Principal, h Heard) Consumer {
|
||||
const ToolsSuffix = "_tools"
|
||||
|
||||
// readerGrants are what a reader publishes to ask about, pull from and acknowledge a durable consumer on a stream: for the
|
||||
// module's program's name and for its tool runner's twin (A-397).
|
||||
// module's program's name and for its tool runner's twin (A-397); and the create and delete of the twin by its name alone,
|
||||
// which the tool runner makes from the membership's spec and removes when the module leaves the node (design 47 H2, H-A-472
|
||||
// fifth to seventh addenda). Every filter goes in FilterSubjects, so the create is on the name, never on a filter.
|
||||
func readerGrants(stream, durable string) []string {
|
||||
var out []string
|
||||
for _, d := range []string{durable, durable + ToolsSuffix} {
|
||||
out = append(out, "$JS.API.CONSUMER.INFO."+stream+"."+d, "$JS.API.CONSUMER.MSG.NEXT."+stream+"."+d, "$JS.ACK."+stream+"."+d+".>")
|
||||
}
|
||||
return out
|
||||
twin := durable + ToolsSuffix
|
||||
return append(out, "$JS.API.CONSUMER.CREATE."+stream+"."+twin, "$JS.API.CONSUMER.DELETE."+stream+"."+twin)
|
||||
}
|
||||
|
||||
// withToolsTwin is a module's consumer and, beside it, the tool runner's twin of it: the same filters under its own name (A-397).
|
||||
func withToolsTwin(c Consumer) []Consumer {
|
||||
// toolsTwin is the tool runner's twin of a module's consumer c: the same filters under its own name (A-397), the events
|
||||
// given up on and delivered again to one consumer alone named by the twin itself.
|
||||
func toolsTwin(c Consumer) Consumer {
|
||||
twin := c
|
||||
twin.Name += ToolsSuffix
|
||||
twin.Filters = append([]string(nil), c.Filters...)
|
||||
// The events given up on and delivered again to one consumer alone are named by it: the twin's are its own.
|
||||
for i, f := range twin.Filters {
|
||||
if f == AgainFilter(c.Name) {
|
||||
twin.Filters[i] = AgainFilter(twin.Name)
|
||||
}
|
||||
}
|
||||
twin.Why = c.Why + "; the tool runner's own, beside the program's (lead answer A-397)"
|
||||
return []Consumer{c, twin}
|
||||
return twin
|
||||
}
|
||||
|
||||
// twinsOf are the consumers the grants part composes of a module's consumer c (design 47 H2, H-A-472 addendum, second and
|
||||
// fifth addenda): c itself, the program's, only for a module with a program of its own (Declared.HasProgram), and never the
|
||||
// tool runner's twin <name>_tools, which the runner creates from the membership's spec at a child's first subscribe.
|
||||
func twinsOf(c Consumer, hasProgram bool) []Consumer {
|
||||
if hasProgram {
|
||||
return []Consumer{c}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardSpec is a heard entry of consumer c's twin: its stream, the module's consumer name the twin is named from, and the
|
||||
// twin's whole spec (H-A-472 seventh addendum).
|
||||
func heardSpec(c Consumer) HeardIssued {
|
||||
twin := toolsTwin(c)
|
||||
policy := membership.DeliverAll
|
||||
switch {
|
||||
case twin.LastPerSubject:
|
||||
policy = membership.DeliverLastPerSubject
|
||||
case twin.FromNow:
|
||||
policy = membership.DeliverFromNow
|
||||
}
|
||||
return HeardIssued{Stream: c.Stream, Consumer: c.Name, Filters: twin.Filters, AckWaitSeconds: twin.AckWaitSeconds,
|
||||
MaxDeliver: twin.MaxDeliver, MaxAckPending: twin.MaxAckPending, DeliverPolicy: policy}
|
||||
}
|
||||
|
||||
// heardGrants are what a principal publishes to read and acknowledge the consumers it hears module streams
|
||||
@@ -334,15 +363,21 @@ func streamsIssuedFor(d Declared) []StreamIssued {
|
||||
return out
|
||||
}
|
||||
|
||||
// HeardIssuedFor is every module stream an assignment hears, as its membership names it.
|
||||
// HeardIssuedFor is every stream an assignment hears, as its membership names it: EVENTS like any stream, each entry with
|
||||
// its consumer's whole spec as the tool runner's twin of it is composed (design 47 H2, H-A-472 seventh addendum), so the
|
||||
// runner computes no name and no filter and the grants part stays the one composer of filters.
|
||||
func HeardIssuedFor(node string, d Declared) []HeardIssued {
|
||||
var out []HeardIssued
|
||||
p := carriedPrincipal(node, d)
|
||||
if c, needed := ConsumerFor(p); needed {
|
||||
out = append(out, heardSpec(c))
|
||||
}
|
||||
// A restricted stream's bind word only for a module of its own account (novox/hq ADR 0376 §1).
|
||||
for _, h := range ComposedHears(d) {
|
||||
for _, h := range p.Hears {
|
||||
if len(h.Filters) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, HeardIssued{Stream: h.Stream, Consumer: node + "_" + d.Module})
|
||||
out = append(out, heardSpec(heardConsumer(p, h)))
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Stream < out[j].Stream })
|
||||
return out
|
||||
|
||||
@@ -156,6 +156,9 @@ type Principal struct {
|
||||
// Process is the process resource's id, for a KindProcess principal alone (novox/hq ADR 0436 §2):
|
||||
// Username and inbox are <node>.<module>.<process id>.
|
||||
Process string
|
||||
// HasProgram is Declared.HasProgram, for a KindModule principal: its program's consumer <node>_<module> is composed
|
||||
// beside the tool runner's twin (H-A-472 second addendum).
|
||||
HasProgram bool
|
||||
|
||||
// Objects are the bus names of its module's own object buckets, and ReadsObjects those of others it reads (novox/hq
|
||||
// ADR 0592 §5, §9).
|
||||
|
||||
@@ -38,7 +38,7 @@ accounts {
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "one.mesh-delivery-watchman", password: "$2a$11$wwwwwwwwwwwwwwwwwwwwww", permissions: {
|
||||
publish: { allow: ["$JS.ACK.DEAD_LETTER_NOTICES.mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman_tools.>", "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.DIRECT.GET.PROV_APPLIED.>", "$JS.API.STREAM.INFO.DEAD_LETTERS", "$JS.API.STREAM.INFO.DEAD_LETTER_NOTICES", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.INFO.PROV_APPLIED", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS", "$JS.API.STREAM.MSG.GET.PROV_APPLIED", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.bus.connz", "mesh.events.dead.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~mesh-delivery-watchman", "mesh.seat.*.call.*.one~mesh-delivery-watchman", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.DEAD_LETTER_NOTICES.mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman_tools.>", "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.CREATE.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.CONSUMER.DELETE.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.DIRECT.GET.PROV_APPLIED.>", "$JS.API.STREAM.INFO.DEAD_LETTERS", "$JS.API.STREAM.INFO.DEAD_LETTER_NOTICES", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.INFO.PROV_APPLIED", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS", "$JS.API.STREAM.MSG.GET.PROV_APPLIED", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.bus.connz", "mesh.events.dead.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~mesh-delivery-watchman", "mesh.seat.*.call.*.one~mesh-delivery-watchman", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$SRV.INFO", "$SRV.INFO.mesh-delivery-watchman", "$SRV.INFO.mesh-delivery-watchman.>", "$SRV.PING", "$SRV.PING.mesh-delivery-watchman", "$SRV.PING.mesh-delivery-watchman.>", "$SRV.STATS", "$SRV.STATS.mesh-delivery-watchman", "$SRV.STATS.mesh-delivery-watchman.>", "_INBOX.one.mesh-delivery-watchman.>", "mesh.bus.connect", "mesh.bus.disconnect", "mesh.control.*.tools-alive", "mesh.events.dead.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.mesh-delivery-watchman.call.>", "mesh.mod.mesh-delivery-watchman.tool.>", "mesh.mod.nats.event.bus.connection-closed", "mesh.mod.nats.event.bus.slow-consumer", "mesh.mod.power.event.*", "mesh.seat.mesh-self-check.event.checked", "prov.applied.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
@@ -47,17 +47,17 @@ accounts {
|
||||
subscribe: { allow: ["_INBOX.one.nats.>"] }
|
||||
} }
|
||||
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.EVENTS.one_telegram_tools.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.telegram", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.telegram", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~telegram", "mesh.seat.*.call.*.one~telegram", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.EVENTS.one_telegram_tools.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.CREATE.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.telegram", "$JS.API.CONSUMER.DELETE.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.telegram", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~telegram", "mesh.seat.*.call.*.one~telegram", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$JS.EVENT.ADVISORY.CONSUMER.CREATED.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.mod.telegram.call.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.ACK.EVENTS.two_audit_tools.>", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.audit", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.INFO.EVENTS.two_audit_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.audit", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~audit", "mesh.seat.*.call.*.two~audit", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.ACK.EVENTS.two_audit_tools.>", "$JS.API.CONSUMER.CREATE.EVENTS.two_audit_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.audit", "$JS.API.CONSUMER.DELETE.EVENTS.two_audit_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.INFO.EVENTS.two_audit_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.audit", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~audit", "mesh.seat.*.call.*.two~audit", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.mod.audit.call.>", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.ACK.EVENTS.two_shop_tools.>", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.shop", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.INFO.EVENTS.two_shop_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.shop", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.mod.shop.event.order.placed", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~shop", "mesh.seat.*.call.*.two~shop", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.accept.send"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.ACK.EVENTS.two_shop_tools.>", "$JS.API.CONSUMER.CREATE.EVENTS.two_shop_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.shop", "$JS.API.CONSUMER.DELETE.EVENTS.two_shop_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.INFO.EVENTS.two_shop_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.shop", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.mod.shop.event.order.placed", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~shop", "mesh.seat.*.call.*.two~shop", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.accept.send"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.mod.shop.call.>", "mesh.mod.shop.tool.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -3,20 +3,75 @@ package buscompose
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
)
|
||||
|
||||
// A-397: beside each module's durable consumer the tool runner's twin, with the same filters under its own name, and the
|
||||
// events delivered again to one consumer named by the twin itself.
|
||||
func TestTheRunnersTwinBesideTheModulesConsumer(t *testing.T) {
|
||||
c := Consumer{Name: "ace_mod", Stream: "EVENTS", Filters: []string{"mesh.mod.x.event.y", AgainFilter("ace_mod")}}
|
||||
got := withToolsTwin(c)
|
||||
if len(got) != 2 || got[0].Name != "ace_mod" || got[1].Name != "ace_mod_tools" {
|
||||
t.Fatalf("consumers = %+v, want ace_mod and ace_mod_tools", got)
|
||||
// Design 47 H2, H-A-472 fifth to seventh addenda: the grants part composes no tool runner's twin; it writes the twin's whole
|
||||
// spec in the membership's heard entries, EVENTS like any stream, and grants the module's reader the create and delete of
|
||||
// its own twin by name.
|
||||
func TestTheRunnersTwinIsTheMembershipsSpec(t *testing.T) {
|
||||
c := Consumer{Name: "ace_mod", Stream: "EVENTS", Filters: []string{"mesh.mod.x.event.y", AgainFilter("ace_mod")}, AckWaitSeconds: 30,
|
||||
MaxDeliver: 5, FromNow: true}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
hasProgram bool
|
||||
want []string
|
||||
}{
|
||||
{"a module with a program of its own: the program's alone", true, []string{"ace_mod"}},
|
||||
{"a module the tool runner carries: none", false, nil},
|
||||
} {
|
||||
var got []string
|
||||
for _, tc := range twinsOf(c, row.hasProgram) {
|
||||
got = append(got, tc.Name)
|
||||
}
|
||||
if !slices.Equal(got, row.want) {
|
||||
t.Errorf("%s: consumers composed = %v, want %v", row.name, got, row.want)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got[1].Filters, []string{"mesh.mod.x.event.y", AgainFilter("ace_mod_tools")}) {
|
||||
t.Errorf("twin filters = %v", got[1].Filters)
|
||||
|
||||
d := Declared{Module: "mod", Consumes: []string{"x.happened"},
|
||||
Hears: []Heard{{Stream: "feed_one", Emitter: "feed", Filters: []string{"mesh.stream.feed.event.a"}},
|
||||
{Stream: "kept_one", Emitter: "keeper", Filters: []string{"mesh.stream.keeper.event.k"}, LastPerSubject: true}}}
|
||||
hears := HeardIssuedFor("ace", d)
|
||||
byStream := map[string]membership.HeardIssued{}
|
||||
for _, h := range hears {
|
||||
byStream[h.Stream] = h
|
||||
if !h.HasSpec() || h.Consumer != "ace_mod" || h.AckWaitSeconds != 30 || h.MaxDeliver != 5 {
|
||||
t.Errorf("heard entry %+v: want the twin's spec under the module's consumer ace_mod", h)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got[0].Filters, c.Filters) {
|
||||
t.Errorf("the module's filters changed: %v", got[0].Filters)
|
||||
for _, row := range []struct {
|
||||
stream, policy string
|
||||
filters []string
|
||||
}{
|
||||
{"EVENTS", membership.DeliverFromNow, []string{"mesh.mod.x.event.happened", AgainFilter("ace_mod_tools")}},
|
||||
{"feed_one", membership.DeliverFromNow, []string{"mesh.stream.feed.event.a"}},
|
||||
{"kept_one", membership.DeliverLastPerSubject, []string{"mesh.stream.keeper.event.k"}},
|
||||
} {
|
||||
h, ok := byStream[row.stream]
|
||||
if !ok {
|
||||
t.Errorf("no heard entry for %s in %+v", row.stream, hears)
|
||||
continue
|
||||
}
|
||||
if h.DeliverPolicy != row.policy || !slices.Equal(h.Filters, row.filters) {
|
||||
t.Errorf("%s: policy %q filters %v, want %q %v", row.stream, h.DeliverPolicy, h.Filters, row.policy, row.filters)
|
||||
}
|
||||
}
|
||||
|
||||
grants := readerGrants("EVENTS", "ace_mod")
|
||||
for _, row := range []struct {
|
||||
subject string
|
||||
want bool
|
||||
}{
|
||||
{"$JS.API.CONSUMER.CREATE.EVENTS.ace_mod_tools", true},
|
||||
{"$JS.API.CONSUMER.DELETE.EVENTS.ace_mod_tools", true},
|
||||
{"$JS.API.CONSUMER.INFO.EVENTS.ace_mod_tools", true},
|
||||
{"$JS.API.CONSUMER.CREATE.EVENTS.ace_mod", false},
|
||||
{"$JS.API.CONSUMER.DELETE.EVENTS.ace_mod", false},
|
||||
} {
|
||||
if slices.Contains(grants, row.subject) != row.want {
|
||||
t.Errorf("reader grants %s: %v, want %v (%v)", row.subject, !row.want, row.want, grants)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,6 +88,11 @@ type Declared struct {
|
||||
// delivered to it and nothing can connect as it (novox/hq issue 195). Said in the negative so a
|
||||
// record that does not say is composed as it always was.
|
||||
NoAccount bool
|
||||
// HasProgram says the module runs a program of its own that reads its consumer <node>_<module>: an account of its own
|
||||
// (an own broker secret) and a process resource with no `bus` of its own (design 47 H2, H-A-472 second addendum).
|
||||
// Without it, nothing is composed on its consumer's name, so the pass makes no orphan; the tool runner's twin
|
||||
// <node>_<module>_tools is the runner's to make from the membership's spec (H-A-472 fifth to seventh addenda).
|
||||
HasProgram bool
|
||||
// SnapshotsTheBus says the module holds mesh-broker — it is the bus — and so is the one module
|
||||
// granted the snapshot API, to copy the bus's streams for the night's backup (novox/hq ADR 0235).
|
||||
SnapshotsTheBus bool
|
||||
@@ -349,7 +354,7 @@ func Users(r Records) ([]Principal, error) {
|
||||
continue
|
||||
}
|
||||
out = append(out, Principal{
|
||||
Kind: KindModule, Node: node, Module: d.Module,
|
||||
Kind: KindModule, Node: node, Module: d.Module, HasProgram: d.HasProgram,
|
||||
Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves,
|
||||
Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes,
|
||||
State: stateNames(d.State), Reads: d.Reads, ReadLogs: d.ReadLogs, SnapshotsTheBus: d.SnapshotsTheBus,
|
||||
|
||||
@@ -4,9 +4,13 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -60,6 +64,89 @@ type heldConsumer struct {
|
||||
Filters []string `json:"filters"`
|
||||
MaxDeliver int `json:"max_deliver"`
|
||||
Pending int64 `json:"pending"`
|
||||
// WaitingPulls and LastDelivered say whether it is read now: pulls waiting on it, and when it last delivered (H-A-472).
|
||||
WaitingPulls int64 `json:"waiting_pulls"`
|
||||
LastDelivered string `json:"last_delivered"`
|
||||
}
|
||||
|
||||
// DefaultTwinCurrentHours is the setting twin-current-hours's default (design 47 H2, H-A-472 third addendum): how many
|
||||
// hours since a twin's last delivery it is current, within the day; the bus module's manifest names the same setting for
|
||||
// nats_consumer_remove, so "current" and "orphaned" share one bound.
|
||||
const DefaultTwinCurrentHours = 24
|
||||
|
||||
// EnvTwinCurrentHours names the file the manifest writes the setting twin-current-hours into.
|
||||
const EnvTwinCurrentHours = "MESH_GRANTS_TWIN_CURRENT_HOURS_FILE"
|
||||
|
||||
// TwinCurrentHoursSetting reads the setting twin-current-hours from its file, read at the start: a missing or empty file
|
||||
// is its default, one that holds no positive whole number an error naming the file.
|
||||
func TwinCurrentHoursSetting(path string) (int, error) {
|
||||
if path == "" {
|
||||
return DefaultTwinCurrentHours, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return DefaultTwinCurrentHours, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return DefaultTwinCurrentHours, nil
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a positive whole number", path, v)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// twinCurrentWithin is the part's bound of a current twin: its setting twin-current-hours, else the default.
|
||||
func (p *Part) twinCurrentWithin() time.Duration {
|
||||
if p.TwinCurrentHours > 0 {
|
||||
return time.Duration(p.TwinCurrentHours) * time.Hour
|
||||
}
|
||||
return DefaultTwinCurrentHours * time.Hour
|
||||
}
|
||||
|
||||
// durableTwin names the twin of a module's durable: its program's <node>_<module> and its tool runner's
|
||||
// <node>_<module>_tools (design 47 G, A-340), each the other's.
|
||||
func durableTwin(name string) string {
|
||||
if t, ok := strings.CutSuffix(name, "_tools"); ok {
|
||||
return t
|
||||
}
|
||||
return name + "_tools"
|
||||
}
|
||||
|
||||
// orphaned says whether a held durable is an orphan (design 47 H2, H-A-472 (2), eleventh addendum): never read at all, or
|
||||
// no pulls waiting on it, while its twin
|
||||
// on the same stream is current, with pulls waiting and a delivery within the bound (the setting twin-current-hours,
|
||||
// the third addendum). A module that moved into the tool runner left its program's durable so, and one that runs its own
|
||||
// program again its tool runner's.
|
||||
func orphaned(h heldConsumer, held map[string]heldConsumer, now time.Time, within time.Duration) bool {
|
||||
if h.WaitingPulls > 0 {
|
||||
return false
|
||||
}
|
||||
// A durable nobody ever read, no pull waiting and no delivery ever, is an orphan whatever its twin's state; one that was
|
||||
// read and lags is behind (H-A-472 eleventh addendum).
|
||||
if neverDelivered(h) {
|
||||
return true
|
||||
}
|
||||
t, ok := held[durableTwin(h.Name)]
|
||||
if !ok || t.WaitingPulls == 0 {
|
||||
return false
|
||||
}
|
||||
last, err := time.Parse(time.RFC3339Nano, t.LastDelivered)
|
||||
return err == nil && now.Sub(last) < within
|
||||
}
|
||||
|
||||
// neverDelivered says the bus never delivered from the durable: no time of a last delivery, or the zero time.
|
||||
func neverDelivered(h heldConsumer) bool {
|
||||
if h.LastDelivered == "" {
|
||||
return true
|
||||
}
|
||||
last, err := time.Parse(time.RFC3339Nano, h.LastDelivered)
|
||||
return err == nil && last.IsZero()
|
||||
}
|
||||
|
||||
// heldConsumers reads the consumers the bus holds on stream, by name.
|
||||
@@ -89,6 +176,25 @@ func heldConsumers(c Caller, stream string) (map[string]heldConsumer, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heardDurablesOf is, by stream, both twins of every heard entry of every assigned module's membership, the plain
|
||||
// `<consumer>` and the tool runner's `<consumer>_tools` (design 47 H2, H-A-472 tenth and eleventh addenda): each one held
|
||||
// and composed by no pass is judged by the behind check.
|
||||
func heardDurablesOf(recs buscompose.Records) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, node := range sortedKeys(recs.Assigned) {
|
||||
for _, d := range recs.Assigned[node] {
|
||||
for _, h := range buscompose.HeardIssuedFor(node, d) {
|
||||
for _, name := range []string{h.Consumer, h.Consumer + buscompose.ToolsSuffix} {
|
||||
if !slices.Contains(out[h.Stream], name) {
|
||||
out[h.Stream] = append(out[h.Stream], name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// filtersToAdd are the composed filters of a consumer the bus's one lacks, in the composed order; none when the consumer
|
||||
// filters the whole stream (no filters) or holds all.
|
||||
func filtersToAdd(c buscompose.Consumer, h heldConsumer) []string {
|
||||
@@ -123,16 +229,50 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
|
||||
st.Wanted = len(wanted)
|
||||
seen := map[string]bool{}
|
||||
defer func() { p.behindSeen = seen }()
|
||||
var orphans []string
|
||||
byStream := map[string][]buscompose.Consumer{}
|
||||
for _, mc := range wanted {
|
||||
byStream[mc.Consumer.Stream] = append(byStream[mc.Consumer.Stream], mc.Consumer)
|
||||
}
|
||||
// The durables no pass composes, the tool runner's twins it makes itself from the membership's spec and the plain one of a
|
||||
// module whose program left, are still judged here, held on the streams each consuming module's heard entries name
|
||||
// (design 47 H2, H-A-472 tenth and eleventh addenda).
|
||||
twins := heardDurablesOf(recs)
|
||||
for stream := range twins {
|
||||
if _, composed := byStream[stream]; !composed {
|
||||
byStream[stream] = nil
|
||||
}
|
||||
}
|
||||
// judge is the behind check of one held consumer: an orphan is left out of it and named in a finding of its own
|
||||
// (H-A-472 (2)), so the alarm says a real backlog only.
|
||||
judge := func(key string, h heldConsumer, held map[string]heldConsumer) {
|
||||
if h.Pending <= BehindBound {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case !p.behindSeen[key]:
|
||||
case orphaned(h, held, p.now(), p.twinCurrentWithin()):
|
||||
orphans = append(orphans, key)
|
||||
default:
|
||||
st.Behind = append(st.Behind, fmt.Sprintf("%s: %d pending", key, h.Pending))
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
for _, stream := range sortedKeys(byStream) {
|
||||
held, err := heldConsumers(p.Call, stream)
|
||||
if err != nil {
|
||||
st.Unread = append(st.Unread, stream+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
composedHere := map[string]bool{}
|
||||
for _, c := range byStream[stream] {
|
||||
composedHere[c.Name] = true
|
||||
}
|
||||
for _, name := range twins[stream] {
|
||||
if h, ok := held[name]; ok && !composedHere[name] {
|
||||
judge(stream+"."+name, h, held)
|
||||
}
|
||||
}
|
||||
for _, c := range byStream[stream] {
|
||||
key := stream + "." + c.Name
|
||||
h, ok := held[c.Name]
|
||||
@@ -140,12 +280,7 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
|
||||
p.sayWorker(key + " is held already")
|
||||
}
|
||||
if ok {
|
||||
if h.Pending > BehindBound {
|
||||
if p.behindSeen[key] {
|
||||
st.Behind = append(st.Behind, fmt.Sprintf("%s: %d pending", key, h.Pending))
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
judge(key, h, held)
|
||||
// A consumer whose module now consumes more events of the stream gains those filters by itself, only ever added:
|
||||
// a filter it holds that is no longer composed stays, and nothing else about it changes.
|
||||
if add := filtersToAdd(c, h); len(add) > 0 && st.On && p.Consumers != nil {
|
||||
@@ -216,6 +351,7 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
|
||||
}
|
||||
p.sayDiffer(st)
|
||||
p.sayBehind(st)
|
||||
p.sayOrphaned(orphans)
|
||||
}
|
||||
|
||||
// sayDiffer raises the warning condition mesh-grants.consumer-differs while an account consumer the part owns (from the
|
||||
@@ -347,6 +483,41 @@ func (p *Part) sayBehind(st *ConsumersState) {
|
||||
}
|
||||
}
|
||||
|
||||
// DurableOrphanedCondition is the key of the warning raised while an account consumer is an orphaned durable: far
|
||||
// behind, no pulls waiting, its twin on the stream current (design 47 H2, H-A-472 (2)).
|
||||
const DurableOrphanedCondition = "mesh-grants.durable-orphaned"
|
||||
|
||||
// sayOrphaned raises the warning condition mesh-grants.durable-orphaned naming the orphaned durables the behind check
|
||||
// left out, once per set, and clears it when none is left (they were removed, by nats_consumer_remove).
|
||||
func (p *Part) sayOrphaned(orphans []string) {
|
||||
if p.Say == nil {
|
||||
return
|
||||
}
|
||||
why := strings.Join(orphans, ", ")
|
||||
p.holdMu.Lock()
|
||||
defer p.holdMu.Unlock()
|
||||
if p.orphanedKnown && why == p.orphanedSaid {
|
||||
return
|
||||
}
|
||||
c := oc.Condition{Key: DurableOrphanedCondition, Subject: oc.ConditionSubject{Scope: "module", ID: "mesh-grants"}}
|
||||
if why == "" {
|
||||
c.State = occ.StateCleared
|
||||
} else {
|
||||
c.Severity, c.State = occ.SeverityWarning, occ.StateRaised
|
||||
c.Summary = fmt.Sprintf("%d durable consumers are orphans: nothing reads them, while their twin on the stream does", len(orphans))
|
||||
c.Headline = "A listener's old consumer is left over"
|
||||
needsYou(&c, "their removal, with the bus module's nats_consumer_remove", "These durables are more than the bound behind "+
|
||||
"their stream's head with no pulls waiting, while their module's twin on the same stream (<node>_<module> and "+
|
||||
"<node>_<module>_tools) has pulls waiting and delivered within "+p.twinCurrentWithin().String()+" (the setting "+
|
||||
"twin-current-hours): a module that moved into or out of the tool runner left them, or a twin the runner made from "+
|
||||
"the membership and no longer reads, and they are not said as messages piling up: "+why)
|
||||
c.Resolved = "no orphaned durable is left"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
p.orphanedSaid, p.orphanedKnown = why, true
|
||||
}
|
||||
}
|
||||
|
||||
// sayWorker logs one named seat worker's outcome (created, held already) once per change, so the journal names each worker the
|
||||
// pass considers (A-413).
|
||||
func (p *Part) sayWorker(line string) {
|
||||
|
||||
@@ -2,6 +2,8 @@ package grants
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -13,6 +15,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -94,9 +97,13 @@ func (p *Part) following(branch string) bool {
|
||||
// revision was taken in as, so grants-for asks the seat mesh-catalog's revision only for a revision it does not hold, never
|
||||
// to learn the newest.
|
||||
type revisionsKept struct {
|
||||
mu sync.Mutex
|
||||
order map[string][]string
|
||||
kept map[string]map[string]catalogue.Manifest
|
||||
mu sync.Mutex
|
||||
order map[string][]string
|
||||
kept map[string]map[string]catalogue.Manifest
|
||||
// grants are the grants each kept revision composes to, by module (grantsOfEntries), with the digest of the inputs they were
|
||||
// composed with: composed once per revision and inputs, kept as long as the revision is (design 47 H2 H-A-478 and its
|
||||
// addenda).
|
||||
grants map[string]keptGrants
|
||||
newest map[string]int64
|
||||
sources map[string]map[int64]int64
|
||||
sourced map[string][]int64
|
||||
@@ -113,6 +120,7 @@ func revisionKey(branch string, r meshcatalog.CatalogRevision) string {
|
||||
func (k *revisionsKept) init() {
|
||||
if k.kept == nil {
|
||||
k.kept, k.order, k.newest = map[string]map[string]catalogue.Manifest{}, map[string][]string{}, map[string]int64{}
|
||||
k.grants = map[string]keptGrants{}
|
||||
k.sources, k.sourced = map[string]map[int64]int64{}, map[string][]int64{}
|
||||
}
|
||||
}
|
||||
@@ -124,6 +132,44 @@ func (k *revisionsKept) get(branch string, r meshcatalog.CatalogRevision) (map[s
|
||||
return m, ok
|
||||
}
|
||||
|
||||
// keptGrants are a revision's composed grants, by module, and the digest of the inputs they were composed with.
|
||||
type keptGrants struct {
|
||||
inputs string
|
||||
grants map[string]map[string]bool
|
||||
}
|
||||
|
||||
// grantsOf is the grants kept of the revision key names, composed with the inputs of the digest given; false when none is.
|
||||
func (k *revisionsKept) grantsOf(key, inputs string) (map[string]map[string]bool, bool) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
g, ok := k.grants[key]
|
||||
if !ok || g.inputs != inputs {
|
||||
return nil, false
|
||||
}
|
||||
return g.grants, true
|
||||
}
|
||||
|
||||
// keepGrants keeps the grants the revision key names composes to with the inputs of the digest given, in place of any kept
|
||||
// before, while that revision is kept: so the grants are bounded as the revisions are (H-A-195's setting).
|
||||
func (k *revisionsKept) keepGrants(key, inputs string, g map[string]map[string]bool) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if _, ok := k.kept[key]; ok {
|
||||
k.grants[key] = keptGrants{inputs: inputs, grants: g}
|
||||
}
|
||||
}
|
||||
|
||||
// inputsDigest is the sha256 of the inputs as JSON: what a revision's kept grants are composed with (H-A-478 second
|
||||
// addendum). Empty when they do not marshal: nothing is kept then.
|
||||
func inputsDigest(in Inputs) string {
|
||||
raw, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (k *revisionsKept) put(branch string, r meshcatalog.CatalogRevision, manifests map[string]catalogue.Manifest, max int) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
@@ -135,6 +181,7 @@ func (k *revisionsKept) put(branch string, r meshcatalog.CatalogRevision, manife
|
||||
k.kept[key] = manifests
|
||||
for len(k.order[branch]) > max {
|
||||
delete(k.kept, k.order[branch][0])
|
||||
delete(k.grants, k.order[branch][0])
|
||||
k.order[branch] = k.order[branch][1:]
|
||||
}
|
||||
if r.Kind != "candidate" && r.Source != 0 {
|
||||
@@ -167,6 +214,7 @@ func boolsOf[V any](m map[string]V) map[string]bool {
|
||||
func (k *revisionsKept) dropLocked(branch string) {
|
||||
for _, key := range k.order[branch] {
|
||||
delete(k.kept, key)
|
||||
delete(k.grants, key)
|
||||
}
|
||||
delete(k.order, branch)
|
||||
delete(k.newest, branch)
|
||||
@@ -250,14 +298,15 @@ func (p *Part) HearCatalogRevision(_ context.Context, data []byte) error {
|
||||
// is answered from what it holds with no read; one it knows but does not hold is read by that number. Otherwise a revision
|
||||
// read whole is kept: once the first page of an answer names a revision that is kept, it is answered from what is kept,
|
||||
// without the pages after it. The branch's newest is read on by the number its first page names, so its pages are of one
|
||||
// revision though a newer one is written between them; a revision whose pages name another is not kept.
|
||||
func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[string]catalogue.Manifest, none bool, err error) {
|
||||
// revision though a newer one is written between them; a revision whose pages name another is not kept. kept names the revision
|
||||
// as it is kept (revisionKey), empty when it is not: what its grants are kept by (H-A-478).
|
||||
func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[string]catalogue.Manifest, kept string, none bool, err error) {
|
||||
branch := in.Branch
|
||||
followed := p.following(branch)
|
||||
newestAsked := in.Group == "" && in.Heads == "" && in.Source == 0 && in.Revision == 0
|
||||
if n, ok := p.revisions.known(in); ok {
|
||||
if m, held := p.revisions.get(branch, meshcatalog.CatalogRevision{Revision: n}); held {
|
||||
return m, false, nil
|
||||
return m, revisionKey(branch, meshcatalog.CatalogRevision{Revision: n}), false, nil
|
||||
}
|
||||
if newestAsked {
|
||||
in.Revision = n
|
||||
@@ -269,14 +318,14 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
|
||||
for {
|
||||
raw, err := p.Call.Call(CallRevision, in)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s does not answer: %w", CallRevision, err)
|
||||
return nil, "", false, fmt.Errorf("%s does not answer: %w", CallRevision, err)
|
||||
}
|
||||
var a meshcatalog.RevisionAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, false, fmt.Errorf("%s answered what is not a revision: %w", CallRevision, err)
|
||||
return nil, "", false, fmt.Errorf("%s answered what is not a revision: %w", CallRevision, err)
|
||||
}
|
||||
if a.NoneYet {
|
||||
return nil, true, nil
|
||||
return nil, "", true, nil
|
||||
}
|
||||
switch {
|
||||
case a.Revision == nil:
|
||||
@@ -286,7 +335,7 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
|
||||
p.revisions.asked(branch, a.Revision.Revision)
|
||||
}
|
||||
if m, ok := p.revisions.get(branch, *a.Revision); ok {
|
||||
return m, false, nil
|
||||
return m, revisionKey(branch, *a.Revision), false, nil
|
||||
}
|
||||
first = a.Revision
|
||||
if newestAsked {
|
||||
@@ -302,15 +351,16 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
|
||||
}
|
||||
m, err := catalogue.Decode(body)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("the manifest of %s in the revision does not decode: %w", e.Module, err)
|
||||
return nil, "", false, fmt.Errorf("the manifest of %s in the revision does not decode: %w", e.Module, err)
|
||||
}
|
||||
manifests[e.Module] = m
|
||||
}
|
||||
if a.Next == "" || a.Next == in.After {
|
||||
if keep && first != nil && followed {
|
||||
p.revisions.put(branch, *first, manifests, p.revisionsPerBranch())
|
||||
kept = revisionKey(branch, *first)
|
||||
}
|
||||
return manifests, false, nil
|
||||
return manifests, kept, false, nil
|
||||
}
|
||||
in.After = a.Next
|
||||
}
|
||||
@@ -389,26 +439,41 @@ func (p *Part) GrantsFor(in meshgrants.GrantsForInput) (meshgrants.GrantsFor, er
|
||||
default:
|
||||
return meshgrants.GrantsFor{}, errors.New("grants-for names a source revision or a run")
|
||||
}
|
||||
cand, none, err := p.revisionEntries(candidate)
|
||||
// Each stage's time in the journal (H-A-478): the inputs, each revision's read and each one's grants, composed or kept.
|
||||
var stages []string
|
||||
stage := func(what string, start time.Time) {
|
||||
stages = append(stages, fmt.Sprintf("%s %s", what, time.Since(start).Round(time.Millisecond)))
|
||||
}
|
||||
defer func() {
|
||||
if p.Logf != nil {
|
||||
p.Logf("grants-for %s: %s", candidateName(candidate), strings.Join(stages, ", "))
|
||||
}
|
||||
}()
|
||||
start := time.Now()
|
||||
cand, candKept, none, err := p.revisionEntries(candidate)
|
||||
stage("the candidate read", start)
|
||||
if err != nil || none {
|
||||
return meshgrants.GrantsFor{Error: orWhy(err, "the candidate's catalog half is not held: the branch has no revision of it")}, nil
|
||||
}
|
||||
newest, none, err := p.revisionEntries(meshcatalog.RevisionInput{Branch: in.Branch})
|
||||
start = time.Now()
|
||||
newest, newestKept, none, err := p.revisionEntries(meshcatalog.RevisionInput{Branch: in.Branch})
|
||||
stage("the newest read", start)
|
||||
if err != nil || none {
|
||||
return meshgrants.GrantsFor{Error: orWhy(err, "the branch's newest catalog half is not held")}, nil
|
||||
}
|
||||
base := map[string]catalogue.Manifest{}
|
||||
for module := range cand {
|
||||
if m, ok := newest[module]; ok {
|
||||
base[module] = m
|
||||
}
|
||||
}
|
||||
start = time.Now()
|
||||
input := p.inputs()
|
||||
was, err := grantsOfEntries(input, base)
|
||||
digest := inputsDigest(input)
|
||||
stage("the inputs", start)
|
||||
start = time.Now()
|
||||
was, composed, err := p.grantsOfRevision(input, digest, newestKept, newest)
|
||||
stage("the newest's grants ("+composed+")", start)
|
||||
if err != nil {
|
||||
return meshgrants.GrantsFor{Error: "the branch's newest does not compose: " + err.Error()}, nil
|
||||
}
|
||||
will, err := grantsOfEntries(input, cand)
|
||||
start = time.Now()
|
||||
will, composed, err := p.grantsOfRevision(input, digest, candKept, cand)
|
||||
stage("the candidate's grants ("+composed+")", start)
|
||||
if err != nil {
|
||||
return meshgrants.GrantsFor{Error: "the candidate does not compose: " + err.Error()}, nil
|
||||
}
|
||||
@@ -435,6 +500,33 @@ func (p *Part) GrantsFor(in meshgrants.GrantsForInput) (meshgrants.GrantsFor, er
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// candidateName names the candidate an ask is of, for the journal.
|
||||
func candidateName(c meshcatalog.RevisionInput) string {
|
||||
if c.Group != "" {
|
||||
return c.Branch + " " + c.Group
|
||||
}
|
||||
return c.Branch + " source " + strconv.FormatInt(c.Source, 10)
|
||||
}
|
||||
|
||||
// grantsOfRevision is the grants a revision's manifests compose to with in (of the digest inputs), by module: those kept of the
|
||||
// revision kept names when they were composed with the same inputs, else composed and kept with it (H-A-478: once per revision
|
||||
// and inputs). how says which.
|
||||
func (p *Part) grantsOfRevision(in Inputs, inputs, kept string, manifests map[string]catalogue.Manifest) (grants map[string]map[string]bool, how string, err error) {
|
||||
if kept != "" && inputs != "" {
|
||||
if g, ok := p.revisions.grantsOf(kept, inputs); ok {
|
||||
return g, "kept", nil
|
||||
}
|
||||
}
|
||||
g, err := grantsOfEntries(in, manifests)
|
||||
if err != nil {
|
||||
return nil, "composed", err
|
||||
}
|
||||
if kept != "" && inputs != "" {
|
||||
p.revisions.keepGrants(kept, inputs, g)
|
||||
}
|
||||
return g, "composed", nil
|
||||
}
|
||||
|
||||
func orWhy(err error, why string) string {
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -332,3 +333,44 @@ func (h Holdings) CapabilitiesLost(last, now membership.Membership) []string {
|
||||
// CapabilityDifference opens what a membership's pass says when it was not published because it would drop a capability
|
||||
// its holder still asks for.
|
||||
const CapabilityDifference = ShadowDifference + "a capability its holder still asks for: "
|
||||
|
||||
// withdrawMemberships deletes the membership of a module no longer assigned on its node (design 47 H2, H-A-472 ninth
|
||||
// addendum): the tool runner there hears the delete and removes the module's `_tools` durables from the streams the entry
|
||||
// named. Only a whole pass withdraws: one that composes part of the mesh, or any of whose reads failed, keeps every key, as
|
||||
// the accounts are kept (accounts.go). A key of the state that is no membership (the bus module's bindings, BoundPrefix)
|
||||
// is never one.
|
||||
func (p *Part) withdrawMemberships(res *Result, in Inputs, composed map[string]Membership) {
|
||||
if in.keepsLoaded() != "" || len(in.ReadFailures()) > 0 {
|
||||
return
|
||||
}
|
||||
keys, err := p.Store.Keys(StateMemberships)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be listed, so no membership of a module gone is withdrawn: %v", StateMemberships, err))
|
||||
return
|
||||
}
|
||||
placed := Placed(in)
|
||||
for _, key := range keys {
|
||||
if _, still := composed[key]; still || strings.HasPrefix(key, BoundPrefix) || placed(key) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := p.Store.Get(StateMemberships, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
bare, err := membership.BareGrantsMembership(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var m membership.Membership
|
||||
if json.Unmarshal(bare, &m) != nil || buscompose.MembershipKey(m.Node, m.Module) != key {
|
||||
continue
|
||||
}
|
||||
if err := p.takeAway(StateMemberships, key); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s, of a module no longer assigned there, cannot be withdrawn: %v", StateMemberships, key, err))
|
||||
continue
|
||||
}
|
||||
if p.Logf != nil {
|
||||
p.Logf("the membership %s is withdrawn: %s is no longer assigned on %s", key, m.Module, m.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,7 +170,8 @@ type Part struct {
|
||||
Emit Emitter
|
||||
// Create creates a declared stream the bus lacks: the process's connection; nil in the tools, which ask the process.
|
||||
Create Creator
|
||||
// Logf logs what no state or condition says (an error of acting on an answer): the process's; nil in the tools.
|
||||
// Logf logs what no state or condition says (an error of acting on an answer): the process's journal, and in the tools
|
||||
// program its standard error (grants-for's stage times, H-A-478).
|
||||
Logf func(string, ...any)
|
||||
// Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools.
|
||||
Objects Objects
|
||||
@@ -197,6 +198,9 @@ type Part struct {
|
||||
// RevisionsPerBranch is the setting grants-for-revisions-per-branch, read at the start (grantsfor.go, design 47 H H-A-195);
|
||||
// 0 is its default.
|
||||
RevisionsPerBranch int
|
||||
// TwinCurrentHours is the setting twin-current-hours, read at the start (consumers.go, design 47 H2, H-A-472 third
|
||||
// addendum): the bound of a current twin, by which a durable is orphaned; 0 is its default.
|
||||
TwinCurrentHours int
|
||||
// Followed is the watched branches the part follows, as named (the branch follower's): grants-for keeps the revisions
|
||||
// of these alone, and frees a branch's once it leaves them; nil in the tools, where every branch is kept.
|
||||
Followed func() []string
|
||||
@@ -230,6 +234,9 @@ type Part struct {
|
||||
behindSaid string
|
||||
behindKnown bool
|
||||
behindSeen map[string]bool
|
||||
// orphanedSaid is the orphaned durables last said as a condition, once orphanedKnown (consumers.go, H-A-472).
|
||||
orphanedSaid string
|
||||
orphanedKnown bool
|
||||
// fullCleared is whether the old history-full key was cleared since the start (full.go, ADR 0595 §6).
|
||||
fullCleared bool
|
||||
// filterRefused is, by consumer, the refusal of the last addition of filters, so a standing refusal is said once.
|
||||
@@ -565,6 +572,7 @@ func (p *Part) Compose() Result {
|
||||
res.Retired[key] = m.Retired
|
||||
}
|
||||
}
|
||||
p.withdrawMemberships(&res, in, ms)
|
||||
p.clearFull()
|
||||
held, heldErr := HeldStreams(p.Call)
|
||||
var ask Ask
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user