Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7be95905f2 | ||
|
|
04fe2e30fe | ||
|
|
52ad5af1e9 | ||
|
|
cd16358b21 | ||
|
|
0c2ead025d | ||
|
|
5ab9bd9acc | ||
|
|
dc33ae26bb | ||
|
|
a096397928 | ||
|
|
5548fff81f | ||
|
|
52fa062e92 | ||
|
|
ef32d60a5e | ||
|
|
cfd2afe39c | ||
|
|
21daeb8893 | ||
|
|
ae1bf2098d | ||
|
|
f09bd2668a | ||
|
|
82156f8128 | ||
|
|
ffa58e51c4 | ||
|
|
cc1034630f | ||
|
|
73cb9ba690 | ||
|
|
fcac1fc5d0 | ||
|
|
db899d8ad4 | ||
|
|
4ca6e478a6 | ||
|
|
564fd5e0bb | ||
|
|
dc6dfef6f0 | ||
|
|
8eee17a368 | ||
|
|
2b79ee1593 | ||
|
|
9931d4f900 |
@@ -154,6 +154,13 @@ func (b *builtEnds) madeOf(module, fp string) map[string]string {
|
||||
return parts
|
||||
}
|
||||
|
||||
// releasedOf is the version released of module's done build at fp, the release whose parts are what its build.ended made (design
|
||||
// 47 G A-421); false when no such release is heard.
|
||||
func (w *world) releasedOf(module, fp string) (string, bool) {
|
||||
k, ok := w.released.withParts(module, w.ends.madeOf(module, fp))
|
||||
return k.Version, ok && k.Version != ""
|
||||
}
|
||||
|
||||
// of is the version of module released from the source revision, with its parts' digests; false before it is heard.
|
||||
func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bool) {
|
||||
r.mu.Lock()
|
||||
|
||||
@@ -656,6 +656,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
// once the first registrations are written (§95, §353).
|
||||
p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf)
|
||||
p.ends.trunk = w.trunkName
|
||||
p.ends.releasedOf = w.releasedOf // a version released before its pipeline passed is taken at the pass (H-A-490)
|
||||
p.ends.runEnded = func(group string) error { return p.dropCandidate(ctx, group) }
|
||||
if err := hearEnds(hub, p.ends); err != nil {
|
||||
logger.Printf("a pipeline's end is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err)
|
||||
|
||||
@@ -409,7 +409,12 @@ func putOfTheBus(t *testing.T) {
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
putModuleRaw(p.w, trunkBranch, "nats", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
|
||||
endDone(t, p.w, "nats", fp, mdigest)
|
||||
// The build and its release (H-A-490 addendum): the put's want names the version released at its parts.
|
||||
built, _ := json.Marshal(builds.BuildEnded{Module: "nats", Fingerprint: fp, Outcome: builds.BuildDone, Manifest: mdigest, Made: map[string]string{"program": "sha256:5"}})
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "nats", Source: 8, Version: "2.10.1", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "nats:program", Digest: "sha256:5"}}})
|
||||
if p.w.ends.put(built) != nil || p.w.released.put(rel) != nil {
|
||||
t.Fatal("the build or its release is not heard")
|
||||
}
|
||||
hkey, _ := nodeengineclient.HaveKey("novox", "nats")
|
||||
before, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
world(buckets.Have, hkey, before)
|
||||
@@ -489,7 +494,7 @@ func putOfTheBus(t *testing.T) {
|
||||
<-woke
|
||||
got = step()
|
||||
w, err := mwc.DecodeWant(wantsKV.values["want.novox.nats"])
|
||||
if got.Step != puts.Wanted || err != nil || w.Fp != fp || !w.Put || !w.Hold {
|
||||
if got.Step != puts.Wanted || err != nil || w.Fp != fp || w.Version != "2.10.1" || !w.Put || !w.Hold {
|
||||
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||
}
|
||||
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
|
||||
|
||||
@@ -153,6 +153,10 @@ type pipelineEnds struct {
|
||||
trunk func() string
|
||||
// now is the clock of the tier progress (tiers.go); nil is time.Now.
|
||||
now func() time.Time
|
||||
// releasedOf is the version released of module's build at fp, matched by the parts its build.ended made (world.releasedOf):
|
||||
// a mover whose version was released before its pipeline passed gets its want at the pass (design 47 H2 H-A-490). nil
|
||||
// matches none.
|
||||
releasedOf func(module, fp string) (string, bool)
|
||||
}
|
||||
|
||||
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
|
||||
@@ -162,7 +166,8 @@ func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipel
|
||||
// take keeps, for each mover of a passed pipeline that released a version, that version as the newest of its module on the
|
||||
// pipeline's branch, by compare-and-set on a newer version (design 47 H §750): an older version, or the same one again, writes
|
||||
// nothing. A passed mover with no version yet is said in the journal and taken when the runner says the end again naming it
|
||||
// (H-A-270); it holds its tier in the pipeline's tier progress meanwhile (H-A-277 (1)). It says whether any key changed.
|
||||
// (H-A-270); it holds its tier in the pipeline's tier progress meanwhile (H-A-277 (1)). A mover whose build's version was
|
||||
// released before the pass is taken at the pass with that version (H-A-490). It says whether any key changed.
|
||||
func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" {
|
||||
return false, nil
|
||||
@@ -185,6 +190,14 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
if m.Fingerprint == "" || !moverPassed(m) {
|
||||
continue
|
||||
}
|
||||
if m.Version == "" && e.releasedOf != nil {
|
||||
// A version released before its pipeline passed (an again pipeline on a built, published key) is never written
|
||||
// on the mover, nor its end said again: it is judged at the pass, matched by its build's parts (H-A-490).
|
||||
if v, ok := e.releasedOf(m.Module, m.Fingerprint); ok {
|
||||
e.logf("%s: %s passed; its version %s was released before the pass, taken now", p.Pipeline, m.Module, v)
|
||||
m.Version = v
|
||||
}
|
||||
}
|
||||
if m.Version == "" {
|
||||
// The runner writes a mover's version at its release's version-released, and says this end again once it does
|
||||
// (H-A-270 (1), (2)): the skip is said, never silent.
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -156,10 +157,15 @@ func TestThePassedVersionIsKept(t *testing.T) {
|
||||
// A6/AC1, A5/AC1 (design 47 H §750): a pass writes, on every node the module is assigned to, the want naming the newest version
|
||||
// released on its branch whose pipeline passed, with the pipeline and the rank; with none passed it writes none. No grant,
|
||||
// canary, provider or composition is asked for. H-A-270 (1): a passed end whose mover names no version yet writes no want and
|
||||
// says so in the journal; the same end said again naming the version writes it.
|
||||
// says so in the journal; the same end said again naming the version writes it. H-A-490: a version released before its pipeline
|
||||
// passed (an again pipeline on a built, published key) gets its want at the pass, matched to the mover's build by its parts.
|
||||
func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
for _, passed := range []bool{false, true} {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
passed, releasedBefore bool
|
||||
}{{"no pipeline passed", false, false}, {"passed, then its version released", true, false}, {"released, then its pipeline passed", true, true}} {
|
||||
passed := c.passed
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
@@ -172,7 +178,16 @@ func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
var journal []string
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
|
||||
p.ends.trunk = p.w.trunkName
|
||||
if passed {
|
||||
p.ends.releasedOf = p.w.releasedOf
|
||||
if c.releasedBefore {
|
||||
// Built and released in an earlier pipeline that failed: the runner's again pipeline ends naming no version.
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "web", Source: 4, Version: "0.1.2", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "web:image", Digest: "sha256:7"}}})
|
||||
built, _ := json.Marshal(builds.BuildEnded{Module: "web", Fingerprint: fp, Outcome: builds.BuildDone, Made: map[string]string{"image": "sha256:7"}})
|
||||
if err := p.w.released.put(rel); err != nil || p.w.ends.put(built) != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
|
||||
} else if passed {
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
|
||||
p.wants(context.Background())
|
||||
if len(wantsKV.values) != 0 || !slices.Contains(journal, trunkBranch+".5: web passed with no version yet: taken again when its version is written") {
|
||||
@@ -185,14 +200,14 @@ func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
raw, got := wantsKV.values[key]
|
||||
if got != passed {
|
||||
t.Errorf("passed %v: a want on %s written = %v", passed, node, got)
|
||||
t.Errorf("%s: a want on %s written = %v", c.name, node, got)
|
||||
continue
|
||||
}
|
||||
if !passed {
|
||||
continue
|
||||
}
|
||||
if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 {
|
||||
t.Errorf("the want on %s = %+v (%v)", node, w, err)
|
||||
t.Errorf("%s: the want on %s = %+v (%v)", c.name, node, w, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,6 +181,7 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
|
||||
if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state
|
||||
w.Build, w.BuildWords = e.Outcome, e.Error
|
||||
}
|
||||
w.Version, _ = p.w.releasedOf(module, put.FP) // matched by its build's parts, as the pass matches a mover's (H-A-490 addendum)
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
|
||||
@@ -34,6 +34,9 @@ type PutWorld struct {
|
||||
Build, BuildWords string
|
||||
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
|
||||
Newest string
|
||||
// Version is the version released at the put's fingerprint, matched by its build's parts as the pass matches a mover's;
|
||||
// "" when none is released (design 47 H2 H-A-490 addendum).
|
||||
Version string
|
||||
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
|
||||
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
|
||||
Composed *ComposedDigest
|
||||
@@ -87,7 +90,10 @@ func StepPut(w PutWorld) PutNext {
|
||||
if h := w.Have; h == nil || h.Fp != p.FP || h.Composed == "" {
|
||||
return say(puts.Refused, "%s runs %s at %s, and its have names no composition at that fingerprint, so the want cannot name what runs", w.Node, w.Module, fp)
|
||||
}
|
||||
p.Composed = w.Have.Composed
|
||||
if w.Version == "" {
|
||||
return say(puts.Refused, "no version released at %s: build or release it first", fp)
|
||||
}
|
||||
p.Composed, p.Version = w.Have.Composed, w.Version
|
||||
words := fmt.Sprintf("the want of %s on %s names %s, the build it runs now (composition %s), from its running statement; the builder's store is not asked",
|
||||
w.Module, w.Node, fp, short(p.Composed))
|
||||
if p.Hold {
|
||||
@@ -115,6 +121,14 @@ func StepPut(w PutWorld) PutNext {
|
||||
default:
|
||||
return say(puts.Build, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
|
||||
}
|
||||
if p.Step != puts.Wanted && p.Step != puts.Holding {
|
||||
// The want names the build's released version beside its fingerprint, which the node-engine composes from (H-A-490
|
||||
// addendum): a build with none released is refused, never wanted by its fingerprint alone.
|
||||
if w.Version == "" {
|
||||
return say(puts.Refused, "no version released at %s: build or release it first", fp)
|
||||
}
|
||||
p.Version = w.Version
|
||||
}
|
||||
if w.Composed == nil || !w.Granted {
|
||||
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
|
||||
if w.Composed != nil {
|
||||
@@ -212,9 +226,10 @@ func needsPerson(w PutWorld, d ComposedDigest) bool {
|
||||
return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot
|
||||
}
|
||||
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint, marked put, and held when it holds.
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint and the version released at it, marked put, and held when
|
||||
// it holds.
|
||||
func PutWant(p puts.Put) mwc.Want {
|
||||
return mwc.Want{Fp: p.FP, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
return mwc.Want{Fp: p.FP, Version: p.Version, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
}
|
||||
|
||||
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
|
||||
|
||||
@@ -105,7 +105,7 @@ func Wants(m Module, wants map[string]mwc.Want) Outcome {
|
||||
if !put && had && before.Put && before.Hold {
|
||||
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard; only a put
|
||||
// heard released gives it back to the rule.
|
||||
p, put = puts.Put{FP: before.Fp, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), Step: puts.Holding}, true
|
||||
p, put = puts.Put{FP: before.Fp, Version: before.Version, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "), Step: puts.Holding}, true
|
||||
}
|
||||
if put {
|
||||
if want, owns := PutOwns(p); owns {
|
||||
|
||||
@@ -92,7 +92,7 @@ func TestAWantNamesTheNewestPassedVersion(t *testing.T) {
|
||||
// want stands, marked put and held, and once released the rule writes the passed version again.
|
||||
func TestAPutOwnsItsNodesWant(t *testing.T) {
|
||||
m := released()
|
||||
put := puts.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||
put := puts.Put{FP: fNew, Version: "1.2.3", Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
step string
|
||||
@@ -100,7 +100,7 @@ func TestAPutOwnsItsNodesWant(t *testing.T) {
|
||||
want *mwc.Want // nil: nothing written on ace
|
||||
}{
|
||||
{"waiting for the operator", puts.Person, wantedOld(), nil},
|
||||
{"wanted", puts.Wanted, wantedOld(), &mwc.Want{Fp: fNew, Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||
{"wanted", puts.Wanted, wantedOld(), &mwc.Want{Fp: fNew, Version: "1.2.3", Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||
{"holding, written already", puts.Holding, map[string]mwc.Want{"ace": PutWant(put)}, nil},
|
||||
{"released: the rule again", puts.Released, map[string]mwc.Want{"ace": PutWant(put)}, &mwc.Want{Fp: fNew, Version: "1.2.3", Pipeline: "main.12", Rank: 2}},
|
||||
{"not heard: a held want at an older fingerprint stands", "", map[string]mwc.Want{"ace": {Fp: fOld, Put: true, Hold: true, Words: "put: back"}}, nil},
|
||||
@@ -144,7 +144,7 @@ func stepsOfAPut(t *testing.T) {
|
||||
running := nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
|
||||
bus := func() PutWorld {
|
||||
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: builds.BuildDone, Newest: fNew,
|
||||
Composed: &d, Granted: true, Have: &running, Now: now}
|
||||
Version: "1.2.3", Composed: &d, Granted: true, Have: &running, Now: now}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -172,6 +172,16 @@ func stepsOfAPut(t *testing.T) {
|
||||
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
|
||||
return w
|
||||
}, puts.Refused, "names no composition", false, false, false},
|
||||
// H-A-490 (addendum): a want names the build's released version, so a done build with none released is refused, and so
|
||||
// is the build the node runs now when no release names its parts.
|
||||
{"a done build with no released version: refused", func() PutWorld { w := bus(); w.Version = ""; return w }, puts.Refused, "no version released at", false, false, false},
|
||||
{"the build the node runs now, no version released: refused", func() PutWorld {
|
||||
w := bus()
|
||||
w.Build, w.Newest, w.Version = "", fpOf('c'), ""
|
||||
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
|
||||
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
|
||||
return w
|
||||
}, puts.Refused, "build or release it first", false, false, false},
|
||||
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = builds.BuildFailed, "go vet"; return w }, puts.Refused, "go vet", false, false, false},
|
||||
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, puts.Compose, "no composition of nats for novox", false, false, false},
|
||||
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, puts.Compose, "does not name novox", false, false, false},
|
||||
@@ -256,6 +266,9 @@ func stepsOfAPut(t *testing.T) {
|
||||
}
|
||||
for _, c := range cases {
|
||||
next := StepPut(c.world())
|
||||
if (next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) && c.world().Put.Step == puts.Asked && next.Put.Version != "1.2.3" {
|
||||
t.Errorf("put, %s: its want names version %q, want the build's released 1.2.3", c.name, next.Put.Version)
|
||||
}
|
||||
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
|
||||
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
|
||||
}
|
||||
|
||||
@@ -65,6 +65,9 @@ type Put struct {
|
||||
V int `json:"v"`
|
||||
// FP is the fingerprint put, of the new form.
|
||||
FP string `json:"fp"`
|
||||
// Version is the version released at FP, which the want names beside it, once the put found it (design 47 H2 H-A-490
|
||||
// addendum).
|
||||
Version string `json:"version,omitempty"`
|
||||
// Hold keeps the want at FP until released.
|
||||
Hold bool `json:"hold,omitempty"`
|
||||
Why string `json:"why"`
|
||||
|
||||
@@ -5,9 +5,12 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -54,3 +57,14 @@ func TestABuildSeatsStepsAreReadInItsHoldersState(t *testing.T) {
|
||||
t.Errorf("a seat with no holder reads %v; want an error naming it", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-487: a build key's time is its entry's own write time, read with the key and no history.
|
||||
func TestABuildKeysTimeIsItsEntrysOwnWriteTime(t *testing.T) {
|
||||
at := time.Date(2026, 11, 22, 20, 45, 0, 0, time.UTC)
|
||||
raw, rev, got, err := entryOf(func(key string) (*stdio.StateEntry, error) {
|
||||
return &stdio.StateEntry{Key: key, Value: json.RawMessage(`{}`), Revision: 4, At: at}, nil
|
||||
})("k")
|
||||
if err != nil || string(raw) != "{}" || rev != 4 || !got.Equal(at) {
|
||||
t.Errorf("entry = %s rev %d at %v (%v); want the entry's own write time %v", raw, rev, got, err, at)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,7 +430,7 @@ func main() {
|
||||
return a, json.Unmarshal(raw, &a)
|
||||
}
|
||||
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr, reserve)
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds").Get), Asked: read(stdio.State("asked")),
|
||||
Newest: newestVia(stdio.Ask), Steps: r}))
|
||||
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
as := stdio.State(versions.ActedState)
|
||||
@@ -481,13 +481,14 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// entryOf reads a key with its revision and when it was written.
|
||||
func entryOf(s stdio.KeptState) func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
// entryOf reads a key with its revision and when it was written, the entry's own write time: no key's history is read
|
||||
// (design 47 H2 H-A-487).
|
||||
func entryOf(get func(key string) (*stdio.StateEntry, error)) func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
return func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
e, err := s.Get(key)
|
||||
e, err := get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, time.Time{}, err
|
||||
}
|
||||
return e.Value, e.Revision, written(s, key, e.Revision), nil
|
||||
return e.Value, e.Revision, e.At, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,6 +233,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
b.PackageWait = time.Duration(packageWait) * time.Minute
|
||||
// A version's digests are recorded under the builder's own bound, never the build's (H-A-492); it is record-bound-seconds.
|
||||
recordBound, err := numberFile(getenv("MESH_BUILDER_RECORD_BOUND_FILE"), int(build.DefaultRecordBound/time.Second))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.RecordBound = time.Duration(recordBound) * time.Second
|
||||
// Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194).
|
||||
b.PublishedVersions = publishedVersionsVia(calls.Call)
|
||||
waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
|
||||
@@ -254,11 +260,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// A package's number reserved is said as the seat mesh-builds' version-reserved, so the indexer re-pins at the reservation (§805).
|
||||
reserved.names = versions.NamesFrom(versions.EntriesOf(branches))
|
||||
reserved.r.Names = reserved.names // a Go package's module path names its tag's directory (§796)
|
||||
reserved.sayReserved = sayReservedOn(js, cred)
|
||||
b.Reserved = reserved
|
||||
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
|
||||
reserved.withdraw = b.Withdraw
|
||||
reserved.recordAgain = b.RecordAgain // a done build's digests its end did not record are recorded at the start (H-A-492)
|
||||
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
|
||||
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
@@ -298,8 +304,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
reserved.pause = publishSet.Pause // a version-released whose tag the forge did not write waits the pause (H-A-484 addendum)
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+16
-16
@@ -289,7 +289,8 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
var tagsWritten []string
|
||||
call := func(tool string, body any) (json.RawMessage, error) {
|
||||
switch tool {
|
||||
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796)
|
||||
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit, never read (H-A-494)
|
||||
tagsWritten = append(tagsWritten, "read "+body.(gitseat.TagsInput).Prefix)
|
||||
return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}})
|
||||
case "seat:" + gitseat.Seat + ".tag":
|
||||
tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit)
|
||||
@@ -360,15 +361,14 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects)
|
||||
}
|
||||
|
||||
// Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another
|
||||
// commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags <directory>/v<version> at
|
||||
// the build's commit, and the module's version-released clears the finding.
|
||||
rv.r.Names = func(context.Context, string, string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/m4", true, nil
|
||||
}
|
||||
// Design 47 H2 H-A-494: no git tag is read before a reserve nor written at a release: m4's 0.2.0, which the seat git would
|
||||
// answer tagged at another commit, is reserved, no finding raised, and its version-released writes no tag.
|
||||
// Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it
|
||||
// (sayReservedOn), and waited on by the seat and the event's name.
|
||||
rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"})
|
||||
rv.names = func(context.Context, string, string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/m4", true, nil
|
||||
}
|
||||
rv.sayReserved = sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"})
|
||||
said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -383,18 +383,18 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
if err := said.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib",
|
||||
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4",
|
||||
Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1})
|
||||
elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere)
|
||||
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised {
|
||||
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised)
|
||||
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.0" || v4.Tag != "" || raised {
|
||||
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.0 naming no tag, no tag read and no finding", v4, err, elsewhere, raised)
|
||||
}
|
||||
select {
|
||||
case m := <-reservedCh:
|
||||
ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data)
|
||||
if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" ||
|
||||
if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.0" ||
|
||||
ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule {
|
||||
t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source"))
|
||||
t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.0, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source"))
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved)
|
||||
@@ -420,11 +420,11 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
}
|
||||
return m
|
||||
}
|
||||
if !rv.heard(ctx, next("m4", "0.2.1")) {
|
||||
if !rv.heard(ctx, next("m4", "0.2.0")) {
|
||||
t.Fatal("the reservations did not take version-released")
|
||||
}
|
||||
if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) {
|
||||
t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten)
|
||||
if _, standing, _ := rv.r.Of(ctx, "m4", "0.2.0"); standing || len(tagsWritten) != 0 {
|
||||
t.Errorf("after version-released m4 0.2.0 standing %v, tags read or written %v: want it released and no tag read or written", standing, tagsWritten)
|
||||
}
|
||||
|
||||
// Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
@@ -57,11 +56,9 @@ type reservations struct {
|
||||
logf func(string, ...any)
|
||||
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
|
||||
withdraw func(module, fingerprint, why string)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
call toolCaller
|
||||
// pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands
|
||||
// it back at once.
|
||||
pause time.Duration
|
||||
// recordAgain records the digests of a done build of module at fp its end did not record, saying so (design 47 H2 H-A-492);
|
||||
// nil records none.
|
||||
recordAgain func(ctx context.Context, module, fp, when string) (string, error)
|
||||
// rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none.
|
||||
rebuild func(ctx context.Context, module, fp, why string) error
|
||||
// say says package-published on the seat mesh-builds; nil says none.
|
||||
@@ -120,7 +117,7 @@ func sayReservedOn(js jetstream.JetStream, cred process.Account) func(context.Co
|
||||
// newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's
|
||||
// verb versions through call (design 47 G §348), the only source numbering reads them from.
|
||||
func newReservations(store versions.ReserveStore, call toolCaller, bounds func() jetstream.KeyValue, found *findings, wake func(context.Context), logf func(string, ...any)) *reservations {
|
||||
rv := &reservations{call: call, bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf}
|
||||
rv := &reservations{bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf}
|
||||
rv.r = &versions.Reserver{Store: store, Filled: rv.filled.Load, ReleasedOf: func(_ context.Context, module string) ([]string, error) {
|
||||
raw, err := call("seat:"+meshcatalog.Seat+".versions", meshcatalog.VersionsInput{Module: module})
|
||||
if err != nil {
|
||||
@@ -137,8 +134,6 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func()
|
||||
return out, nil
|
||||
}}
|
||||
rv.r.Took, rv.r.Logf, rv.r.Reserved = rv.took, logf, rv.reservedSaid
|
||||
// A module's tags are read before it reserves, and a number tagged at another commit skipped (design 47 H2 §796).
|
||||
rv.r.Tags, rv.r.TaggedElsewhere = tagsVia(call), rv.taggedElsewhere
|
||||
return rv
|
||||
}
|
||||
|
||||
@@ -313,15 +308,9 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err)
|
||||
break
|
||||
}
|
||||
if refused, err := rv.release(ctx, ev); err != nil {
|
||||
if err := rv.release(ctx, ev); err != nil {
|
||||
rv.logf("%v; heard again", err)
|
||||
// A tag the seat git's holder did not write is handed back with the publish pause, so the deliveries the consumer
|
||||
// allows span the holder's outage (design 47 H2 H-A-484 addendum).
|
||||
nak := m.Nak
|
||||
if refused && rv.pause > 0 {
|
||||
nak = func() error { return m.NakWithDelay(rv.pause) }
|
||||
}
|
||||
if nakErr := nak(); nakErr != nil {
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf)
|
||||
@@ -336,16 +325,12 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// release takes a released version in as the catalog's version-released says it (design 47 G §369, §370): its tag written, its
|
||||
// reservation released, its bound and publish item ended and its findings cleared. refused says the tag was not written (the
|
||||
// seat git did not write it, or the reservation could not be read to tag it); the reservation then stands.
|
||||
func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) (refused bool, err error) {
|
||||
if err := rv.tagged(ctx, ev); err != nil {
|
||||
return true, err
|
||||
}
|
||||
// release takes a released version in as the catalog's version-released says it (design 47 G §369): its reservation released,
|
||||
// its bound and publish item ended and its findings cleared. No git tag is written (design 47 H2 H-A-494).
|
||||
func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error {
|
||||
gone, err := rv.r.Released(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return err
|
||||
}
|
||||
for _, k := range gone {
|
||||
rv.dropBound(ctx, k)
|
||||
@@ -355,20 +340,29 @@ func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionRelea
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
return false, nil
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) // one raised before H-A-494
|
||||
return nil
|
||||
}
|
||||
|
||||
// releasedAtStart takes in, at the builder's start, every built, unreleased reservation whose version the catalog's verb versions
|
||||
// holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one
|
||||
// said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended.
|
||||
// said while the builder was away, is taken all the same, its reservation released and its publish item ended; no tag is written
|
||||
// (design 47 H2 H-A-494).
|
||||
// A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released.
|
||||
// Before that, a standing reservation with no digests recorded whose build key is done has them recorded from the done key
|
||||
// (design 47 H2 H-A-492), so it is held against the released versions too.
|
||||
func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
all, err := rv.r.All(ctx)
|
||||
if err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
if rv.recordedAtStart(ctx, all) {
|
||||
if all, err = rv.r.All(ctx); err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
read := map[string][]string{}
|
||||
for _, v := range builtOf(all) {
|
||||
if versions.NotStanding(v, true) != "" {
|
||||
@@ -384,7 +378,7 @@ func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
if !slices.Contains(released, v.Version) {
|
||||
continue
|
||||
}
|
||||
if _, err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil {
|
||||
if err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil {
|
||||
rv.logf("%s %s is released in the catalog and not taken in at the start: %v", v.Module, v.Version, err)
|
||||
continue
|
||||
}
|
||||
@@ -392,6 +386,26 @@ func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// recordedAtStart records the digests of every standing reservation that has none recorded and whose build key is done (design 47
|
||||
// H2 H-A-492): a record its build's end lost. It answers whether one was recorded; one that cannot be is said.
|
||||
func (rv *reservations) recordedAtStart(ctx context.Context, all []versions.ReservedVersion) bool {
|
||||
if rv.recordAgain == nil {
|
||||
return false
|
||||
}
|
||||
recorded := false
|
||||
for _, v := range all {
|
||||
if !v.BuiltAt.IsZero() || v.Fingerprint == "" || versions.NotStanding(v, true) != "" {
|
||||
continue
|
||||
}
|
||||
version, err := rv.recordAgain(ctx, v.Module, v.Fingerprint, "at the builder's start")
|
||||
if err != nil {
|
||||
rv.logf("%v", err)
|
||||
}
|
||||
recorded = recorded || version != ""
|
||||
}
|
||||
return recorded
|
||||
}
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told
|
||||
// before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told
|
||||
// nothing.
|
||||
@@ -451,33 +465,6 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str
|
||||
return fmt.Sprintf("the reservation of %s %s is dropped (%s, %s)", module, series, by, why), nil
|
||||
}
|
||||
|
||||
// tagged writes the git tag of a released version (design 47 G §370): on the catalog's version-released, through the seat
|
||||
// git's verb tag, at the commit the reservation of that version was made from. A version with no reservation (a superseded
|
||||
// build's version is never released, and the key of a version released by another builder is gone) or one that names no
|
||||
// tag (a module with no Go package) is tagged nowhere. A tag the forge could not write is an error, the message heard again.
|
||||
func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error {
|
||||
res, found, err := rv.r.Of(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the reservation of %s %s cannot be read to tag it: %w", ev.Module, ev.Version, err)
|
||||
}
|
||||
if !found || res.Tag == "" || res.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
raw, err := rv.call("seat:"+git.Seat+".tag", git.TagInput{Repository: res.Repository, Commit: res.Commit, Tag: res.Tag})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the tag %s of %s is not written: %w", res.Tag, ev.Module, err)
|
||||
}
|
||||
var a git.TagAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return fmt.Errorf("the git seat answered what is no tag answer for %s: %w", res.Tag, err)
|
||||
}
|
||||
if a.Error != "" || !a.Performed {
|
||||
return fmt.Errorf("the tag %s of %s is not written: %s (%s)", res.Tag, ev.Module, a.Error, a.ErrorKind)
|
||||
}
|
||||
rv.logf("%s is tagged %s at %s", ev.Module, res.Tag, res.Commit)
|
||||
return nil
|
||||
}
|
||||
|
||||
// noMark says why a published marked nothing, from the reservations the published read (design 47 H H-A-280: no second read):
|
||||
// what each holds of digests, and how many stand at that version by the index by version (its one key, no reservation read).
|
||||
func (rv *reservations) noMark(ctx context.Context, o versions.Outcome, read []versions.ReadReservation) string {
|
||||
@@ -549,19 +536,6 @@ func (rv *reservations) heldElsewhere(ctx context.Context, module, why string) {
|
||||
needs: "builder_reserved shows the reservation and its withdrawn numbers; the registry seat's versions shows what it holds."})
|
||||
}
|
||||
|
||||
// taggedElsewhere raises version-tagged-elsewhere about module (design 47 H2 §796): numbers whose git tag stands at another
|
||||
// commit were skipped at reserving; a tag is never moved. Cleared at the module's next version-released.
|
||||
func (rv *reservations) taggedElsewhere(ctx context.Context, module string, skipped []string) {
|
||||
why := "the tags of " + strings.Join(skipped, ", ") + " stand at another commit, so the reserve skipped them"
|
||||
rv.logf("%s: %s", module, why)
|
||||
rv.found.raise(ctx, finding{kind: versions.VersionTaggedElsewhere, scope: meshconditionsclient.ScopeModule, id: module,
|
||||
summary: module + ": " + why,
|
||||
headline: "A number whose tag stands at another commit is not used",
|
||||
explanation: "The repository holds a tag of " + module + " at a commit other than the build's: " + why + ". A tag is never moved.",
|
||||
resolved: "The catalog says version-released for " + module + ".",
|
||||
needs: "builder_reserved shows the reservation; the seat git's tags shows each tag with its commit."})
|
||||
}
|
||||
|
||||
// movedElsewhere takes a done build whose number a registry holds at another digest off that number (design 47 H H-A-275,
|
||||
// H-A-279): every artifact carries its version from its build, so no digest survives a move. The number is withdrawn as a gap
|
||||
// (H-A-226), the finding raised naming both digests, and the build of the same fingerprint asked again (the build-again path of a
|
||||
|
||||
@@ -156,123 +156,117 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom: design 47 G §370 — the git tag go/vMAJOR.MINOR.PATCH is written through
|
||||
// the seat git's tag on the catalog's version-released, at the commit the version's reservation was made from; a module with no
|
||||
// Go package, and a version never released (a superseded build's), are tagged nowhere; a tag the forge did not write leaves
|
||||
// version-released to be heard again with the reservation standing.
|
||||
func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
// TestAReleasedVersionWritesNoTag: design 47 H2 H-A-494 — the builder writes no git tag at a release, and numbers a series from the
|
||||
// catalog's versions and the registries' alone: the catalog's version-released releases the reservation with no call of the seat
|
||||
// git, a reservation written before (one naming a tag) included; the start's take-in of a version the catalog holds released
|
||||
// (H-A-484 c) the same; and a reserve reads no git tag, a tag standing at another commit no reason to skip a number.
|
||||
func TestAReleasedVersionWritesNoTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var tags []git.TagInput
|
||||
refuse := ""
|
||||
var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796)
|
||||
var gitCalls []string // every call of the seat git: none is made
|
||||
var catalog []meshcatalog.VersionEntry // the released versions the catalog's verb versions answers
|
||||
call := func(addr string, body any) (json.RawMessage, error) {
|
||||
switch addr {
|
||||
case "seat:git.tags":
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true})
|
||||
case "seat:git.tag":
|
||||
default:
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog})
|
||||
if strings.HasPrefix(addr, "seat:git.") {
|
||||
gitCalls = append(gitCalls, addr)
|
||||
if addr == "seat:git.tags" { // a tag standing at another commit, as the forge may still hold one
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}}})
|
||||
}
|
||||
return json.Marshal(git.TagAnswer{Performed: true})
|
||||
}
|
||||
tags = append(tags, body.(git.TagInput))
|
||||
return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse})
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog})
|
||||
}
|
||||
fw := &memFindings{vals: map[string][]byte{}}
|
||||
found := &findings{module: builds.BuilderModule, w: fw, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
store := &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
rv := newReservations(store, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
// Design 47 H2 §796: the tag names the Go module's directory, read from its module path: mesh-sdk's `go` here.
|
||||
rv.r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/go", artifact == "lib" || artifact == "pkg", nil
|
||||
rv.names = func(_ context.Context, _, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/go", artifact == "lib", nil
|
||||
}
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true, GoArtifact: "lib"}
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rv.Reserve(ctx, "prog", "2.0", []string{"prog:bin"}, versions.Source{Repository: "o/p", Commit: "beef"}); err != nil {
|
||||
t.Fatal(err)
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", Fingerprint: "fpA"}
|
||||
// oldRecord rewrites a reservation as one written before H-A-494, naming the tag its release was to be written as.
|
||||
oldRecord := func(module, series, tag string) {
|
||||
key := versions.ReservedKey(module, series)
|
||||
var v versions.ReservedVersion
|
||||
if err := json.Unmarshal(store.vals[key], &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.Tag = tag
|
||||
store.vals[key], _ = json.Marshal(v)
|
||||
}
|
||||
release := func(module, version string) *wakeMsg {
|
||||
m := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"` + module + `","version":"` + version + `","source":1,"artifacts":[]}`)}
|
||||
rv.heard(ctx, m)
|
||||
return m
|
||||
}
|
||||
if m := release("prog", "2.0.0"); !m.acked || len(tags) != 0 {
|
||||
t.Fatalf("a module with no Go package: acked %v, tags %v; want none written", m.acked, tags)
|
||||
// The release path: a Go package's version-released releases it, acknowledged, with no tag written.
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil || v.Tag != "" {
|
||||
t.Fatalf("reserved %+v (%v), want a reservation naming no tag", v, err)
|
||||
}
|
||||
refuse = "forge down"
|
||||
if m := release("pkg", "0.1.0"); !m.naked || m.acked {
|
||||
t.Fatalf("a tag the forge did not write: naked %v acked %v; want the release heard again", m.naked, m.acked)
|
||||
oldRecord("pkg", "0.1", "go/v0.1.0")
|
||||
if m := release("pkg", "0.1.0"); !m.acked || m.naked {
|
||||
t.Fatalf("the release: acked %v naked %v; want it acknowledged", m.acked, m.naked)
|
||||
}
|
||||
// Design 47 H2 H-A-484 addendum: with a publish pause set, a refused tag hands the version-released back after the pause, so
|
||||
// the deliveries its consumer allows span the holder's outage.
|
||||
rv.pause = 30 * time.Second
|
||||
if m := release("pkg", "0.1.0"); m.delayed != 30*time.Second || m.naked || m.acked {
|
||||
t.Fatalf("a tag the forge did not write, with the pause: delayed %v naked %v acked %v; want handed back after 30s", m.delayed, m.naked, m.acked)
|
||||
if len(gitCalls) != 0 {
|
||||
t.Errorf("the release called the seat git: %v; want no tag written", gitCalls)
|
||||
gitCalls = nil
|
||||
}
|
||||
rv.pause = 0
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("the reservation went with an unwritten tag: %v", all)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 {
|
||||
t.Fatalf("the released version still stands: %v", all)
|
||||
}
|
||||
// Design 47 H2 H-A-484 addendum: at the builder's start, a built reservation the catalog holds released is tagged and released
|
||||
// as its version-released would (a version-released the bus gave up on), and one the catalog does not hold, or not built,
|
||||
// stands.
|
||||
refuse = ""
|
||||
tags = nil
|
||||
// The start check (H-A-484 c): a built reservation the catalog holds released is released at the start, no tag written; one
|
||||
// not built, or not released, stands.
|
||||
// Design 47 H2 H-A-492: at the start, a standing reservation with no digests recorded whose build key is done has them
|
||||
// recorded (the builder's RecordAgain), and is then held against the released versions as a built one is.
|
||||
if _, err := rv.Reserve(ctx, "rec", "0.1", []string{"rec:bin"}, versions.Source{Repository: "o/q", Commit: "abc", Fingerprint: "fpR"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var recordedAt []string
|
||||
rv.recordAgain = func(ctx context.Context, module, fp, when string) (string, error) {
|
||||
recordedAt = append(recordedAt, module+" "+fp+" "+when)
|
||||
_, err := rv.r.Built(ctx, module, "0.1.0", "rec:bin", "sha256:bb", nil) // its done key's digest
|
||||
return "0.1.0", err
|
||||
}
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "rec", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if _, standing, _ := rv.r.Of(ctx, "rec", "0.1.0"); standing || !slices.Equal(recordedAt, []string{"rec fpR at the builder's start"}) {
|
||||
t.Fatalf("at the start, rec 0.1.0 unrecorded and released: standing %v, recorded %q; want it recorded once at the start and released", standing, recordedAt)
|
||||
}
|
||||
catalog, rv.recordAgain = nil, nil
|
||||
var ended []string
|
||||
rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) }
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", Fingerprint: "fpB"}); err != nil {
|
||||
t.Fatal(err) // 0.1.0 again: the stub catalog holds none released
|
||||
}
|
||||
oldRecord("pkg", "0.1", "go/v0.1.0")
|
||||
rv.releasedAtStart(ctx) // built nothing yet: nothing taken
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 {
|
||||
t.Fatalf("at the start with no build done: reservations %v, tags %+v; want the reservation standing, no tag", all, tags)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("at the start with no build done: reservations %v; want the reservation standing", all)
|
||||
}
|
||||
if _, err := rv.r.Built(ctx, "pkg", "0.1.0", "pkg:lib", "sha256:aa", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.releasedAtStart(ctx) // the catalog holds none released: it stands
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 {
|
||||
t.Fatalf("at the start with nothing released: reservations %v, tags %+v; want the reservation standing", all, tags)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("at the start with nothing released: reservations %v; want the reservation standing", all)
|
||||
}
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "pkg", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) || !slices.Equal(ended, []string{"pkg 0.1.0"}) {
|
||||
t.Fatalf("at the start with 0.1.0 released: reservations %v, tags %+v, items ended %v; want it tagged, released and its item ended", all, tags, ended)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 || !slices.Equal(ended, []string{"pkg 0.1.0"}) {
|
||||
t.Fatalf("at the start with 0.1.0 released: reservations %v, items ended %v; want it released and its item ended", all, ended)
|
||||
}
|
||||
if len(gitCalls) != 0 {
|
||||
t.Errorf("the start check called the seat git: %v; want no tag written", gitCalls)
|
||||
gitCalls = nil
|
||||
}
|
||||
catalog, rv.released = nil, nil
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // the version is reserved again for the rows below
|
||||
t.Fatal(err)
|
||||
}
|
||||
tags = nil
|
||||
if m := release("pkg", "0.1.0"); !m.acked || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) {
|
||||
t.Fatalf("the release: acked %v, tags %+v; want go/v0.1.0 at c0ffee of o/r", m.acked, tags)
|
||||
}
|
||||
tags = nil
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // build A: 0.1.0 (the stub catalog holds none released)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true, GoArtifact: "lib"}); err != nil || v.Version != "0.1.1" { // build B supersedes A
|
||||
t.Fatalf("second build reserved %q (%v)", v.Version, err)
|
||||
}
|
||||
release("pkg", "0.1.0") // A's version is never released in fact; were it said, it is no reservation of the key
|
||||
if len(tags) != 0 {
|
||||
t.Fatalf("a superseded build's version was tagged: %+v", tags)
|
||||
}
|
||||
if release("pkg", "0.1.1"); len(tags) != 1 || tags[0].Commit != "b0b" {
|
||||
t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags)
|
||||
}
|
||||
// Design 47 H2 §796: a tag standing at another commit makes the next number and the finding version-tagged-elsewhere, which
|
||||
// clears at the module's next version-released.
|
||||
standing = []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}, {Tag: "go/v0.2.1", Commit: "elsewhere"}}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.2" || v.Tag != "go/v0.2.2" {
|
||||
t.Fatalf("reserved %+v (%v) with 0.2.0 and 0.2.1 tagged elsewhere, want 0.2.2 tagged go/v0.2.2", v, err)
|
||||
// Numbering reads no tag: the seat git would answer go/v0.2.0 at another commit, and 0.2.0 is reserved all the same.
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.0" || v.Tag != "" || len(gitCalls) != 0 {
|
||||
t.Errorf("reserved %+v (%v), git calls %v; want 0.2.0 naming no tag, no git tag read", v, err, gitCalls)
|
||||
}
|
||||
key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "pkg", versions.VersionTaggedElsewhere)
|
||||
if got, err := decodeFinding(fw.vals[key]); err != nil || !strings.Contains(got.Summary, "0.2.0") || !strings.Contains(got.Summary, "0.2.1") {
|
||||
t.Fatalf("the finding %s = %+v (%v), want it naming 0.2.0 and 0.2.1", key, got, err)
|
||||
if fw.vals[key] != nil {
|
||||
t.Errorf("a reserve raised %s", key)
|
||||
}
|
||||
if release("pkg", "0.2.2"); fw.vals[key] != nil {
|
||||
t.Errorf("version-released of pkg left %s standing", key)
|
||||
}
|
||||
standing = nil
|
||||
// Design 47 H2 §805, H-A-420 (2): a package's number newly reserved is said as version-reserved, its path the package's name
|
||||
// and its revision the indexer's `<branch>.<n>`; a reservation kept as it stood, an artifact that is no package and a
|
||||
// reservation naming no catalog revision say none.
|
||||
@@ -281,7 +275,7 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
return "git.example/o/r/" + artifact, module == "lib2" && artifact == "pkg", nil
|
||||
}
|
||||
rv.sayReserved = func(_ context.Context, ev meshbuilds.VersionReservedEvent) error { said = append(said, ev); return nil }
|
||||
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", GoPackage: true, GoArtifact: "pkg", Branch: "main", CatalogRevision: 158}
|
||||
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", Branch: "main", CatalogRevision: 158}
|
||||
for range 2 {
|
||||
if _, err := rv.Reserve(ctx, "lib2", "0.3", []string{"lib2:pkg", "lib2:bin"}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -19,6 +19,8 @@ import (
|
||||
// `module` alone the module's keys newest first, at most MostBuilds, each with its fingerprint whole, its state, revision and
|
||||
// since when, its step and node, its error whole, event-builds and the open ask (§568), and `newest` naming each watched branch
|
||||
// whose newest catalog graph entry of the module names that fingerprint, read from the catalog half by the tools bundle.
|
||||
// **Each key's time is its entry's own write time** (design 47 H2 H-A-487): the history of a key is read only for the one key a
|
||||
// fingerprint names, when its entry does not say when it was written, and a listing cut at MostBuilds says so in its note.
|
||||
|
||||
// BuildTool is the builder's read tool of a module's build keys.
|
||||
const BuildTool = "builder_build"
|
||||
@@ -56,14 +58,16 @@ type BuildAnswer struct {
|
||||
Builds []BuildRow `json:"builds"`
|
||||
// Newest is the fingerprint of the module's newest catalog graph entry on each watched branch.
|
||||
Newest map[string]string `json:"newest-entries,omitempty"`
|
||||
// Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint.
|
||||
// Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint, and that the listing
|
||||
// was cut at MostBuilds (H-A-487).
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// BuildReader reads what builder_build answers.
|
||||
type BuildReader struct {
|
||||
Builds StateRead
|
||||
// Entry is a key's value with its revision and when it was written; nil leaves both out.
|
||||
// Entry is a key's value with its revision and when it was written, the entry's own write time (H-A-487); nil leaves both
|
||||
// out. Builds.Written, the key's history, is read only for the key a fingerprint names when Entry says no time.
|
||||
Entry func(key string) (json.RawMessage, uint64, time.Time, error)
|
||||
// Asked is the builder's state asked, which holds the asks the operator was asked.
|
||||
Asked StateRead
|
||||
@@ -96,6 +100,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
if raw == nil {
|
||||
continue
|
||||
}
|
||||
if at.IsZero() && fingerprint != "" && r.Builds.Written != nil {
|
||||
at = r.Builds.Written(key) // one key's history, never every key's (H-A-487)
|
||||
}
|
||||
v, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%s holds what is not a build: %w", key, err)
|
||||
@@ -118,7 +125,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
}
|
||||
return rows[i].Key > rows[j].Key
|
||||
})
|
||||
var cut string
|
||||
if len(rows) > MostBuilds {
|
||||
cut = fmt.Sprintf("%d of %s's %d build keys are answered, the newest by write time; name a fingerprint for another", MostBuilds, module, len(rows))
|
||||
rows = rows[:MostBuilds]
|
||||
}
|
||||
asks := r.openAsks()
|
||||
@@ -149,6 +158,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
out.Note = "no build key of " + module + " is the newest entry's: " + strings.Join(says, "; ")
|
||||
}
|
||||
}
|
||||
if cut != "" {
|
||||
out.Note = strings.TrimPrefix(out.Note+"; "+cut, "; ")
|
||||
}
|
||||
out.Builds = append(out.Builds, rows...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -66,6 +67,31 @@ func TestBuilderBuildAnswersAModulesKeysWhole(t *testing.T) {
|
||||
if n, err := r.Build("m", ""); err != nil || !strings.Contains(n.Note, "no build key of m is the newest entry's") || !strings.Contains(n.Note, "main names src2:"+strings.Repeat("c", 64)) {
|
||||
t.Errorf("a module none of whose keys the newest entry names: %+v, %v", n, err)
|
||||
}
|
||||
|
||||
// H-A-487: a module of 100 keys is read by each entry's own write time, no key's history read, and the listing is cut at the
|
||||
// 20 newest, saying so; a fingerprint's key with no time in its entry has its one history read.
|
||||
many, at := map[string][]byte{}, map[string]time.Time{}
|
||||
base := time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC)
|
||||
for i := range 100 {
|
||||
k, _ := builds.BuilderBuildKey("big", fmt.Sprintf("src2:%064x", i))
|
||||
many[k], at[k] = done, base.Add(time.Duration(i)*time.Minute)
|
||||
}
|
||||
histories := 0
|
||||
st := stateOf(many)
|
||||
st.Written = func(string) time.Time { histories++; return base }
|
||||
big := &BuildReader{Builds: st, Entry: func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, at[k], nil }}
|
||||
a, err = big.Build("big", "")
|
||||
if err != nil || len(a.Builds) != MostBuilds || !a.Builds[0].Since.Equal(base.Add(99*time.Minute)) || histories != 0 ||
|
||||
!strings.Contains(a.Note, "20 of big's 100 build keys are answered") {
|
||||
t.Fatalf("100 keys: %d rows, newest %v, %d histories read, note %q; want the 20 newest, no history, the cut said", len(a.Builds), a.Builds[0].Since, histories, a.Note)
|
||||
}
|
||||
big.Entry = func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, time.Time{}, nil }
|
||||
if a, err := big.Build("big", ""); err != nil || len(a.Builds) != MostBuilds || histories != 0 {
|
||||
t.Fatalf("100 keys with no time in their entries: %d rows (%v), %d histories read; want no history read for a listing", len(a.Builds), err, histories)
|
||||
}
|
||||
if one, err := big.Build("big", fmt.Sprintf("src2:%064x", 7)); err != nil || len(one.Builds) != 1 || !one.Builds[0].Since.Equal(base) || histories != 1 {
|
||||
t.Errorf("one fingerprint with no time in its entry: %+v (%v), %d histories; want its one history read", one, err, histories)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §589: builder_stuck answers each key's fingerprint whole, as the build-input fingerprint the catalog graph entry carries it.
|
||||
|
||||
@@ -170,6 +170,7 @@ func Unreferenced(b builds.BuilderBuild) bool {
|
||||
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
|
||||
// which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
|
||||
// another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder);
|
||||
// - a failed key withdrawn (withdrawnPrefix) is final for its fingerprint, taken again only by a person's build again (H-A-489);
|
||||
// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is
|
||||
// published (ClaimOptions.Published, ADR 0500 clause 6), and not before;
|
||||
// - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its
|
||||
@@ -229,6 +230,8 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)):
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, withdrawnPrefix) && !opt.ByHand:
|
||||
return nil, nil // withdrawn is final for its fingerprint: no look builds it again (H-A-489)
|
||||
case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver && !opt.ByEvent:
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
|
||||
|
||||
@@ -106,11 +106,19 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
|
||||
return nil
|
||||
}
|
||||
raw, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
if err != nil {
|
||||
b.logf("%s: the build key cannot be read, so it is taken as not done: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
cur, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil || cur.State != builds.BuildDone {
|
||||
if err != nil {
|
||||
b.logf("%s: the build key holds what is no build, so it is taken as not done: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if cur.State != builds.BuildDone {
|
||||
return nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
@@ -122,6 +130,57 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
|
||||
return out
|
||||
}
|
||||
|
||||
// DefaultRecordBound is the bound of the record of a reservation's digests, its setting record-bound-seconds' default (design 47
|
||||
// H2 H-A-492).
|
||||
const DefaultRecordBound = time.Minute
|
||||
|
||||
// recordContext is the context a reservation's digests are recorded under: ctx's values with no cancel of its own, bounded by
|
||||
// RecordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record.
|
||||
func (b *Builder) recordContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
bound := b.RecordBound
|
||||
if bound <= 0 {
|
||||
bound = DefaultRecordBound
|
||||
}
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), bound)
|
||||
}
|
||||
|
||||
// RecordAgain records the digests of fp's done build under its standing reservation when its end did not record them (design 47
|
||||
// H2 H-A-492): at the builder's look of the key and at its start (when says which), said each time. It answers the version
|
||||
// recorded, "" when nothing was to be recorded: the key is not done, no reservation stands for fp, or it is recorded already.
|
||||
func (b *Builder) RecordAgain(ctx context.Context, module, fp, when string) (string, error) {
|
||||
b.init()
|
||||
if b.Reserved == nil {
|
||||
return "", nil
|
||||
}
|
||||
digests := b.doneDigests(ctx, module, fp)
|
||||
if digests == nil {
|
||||
return "", nil
|
||||
}
|
||||
res, ok, err := b.Reserved.ReservedFor(ctx, module, fp)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s at %s: whether its done build's digests are recorded cannot be read %s: %w", module, fp, when, err)
|
||||
}
|
||||
if !ok || !res.BuiltAt.IsZero() || len(res.Artifacts) == 0 || versions.NotStanding(res, true) != "" {
|
||||
return "", nil
|
||||
}
|
||||
refs := map[string]string{}
|
||||
for ref := range res.Artifacts {
|
||||
_, artifact, _ := strings.Cut(ref, ":")
|
||||
d, have := digests[artifact]
|
||||
if !have {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded, and the done key holds no digest of %s to record %s", module, fp, res.Version, ref, when)
|
||||
}
|
||||
refs[ref] = d
|
||||
}
|
||||
rec, stop := b.recordContext(ctx)
|
||||
defer stop()
|
||||
if err := b.Reserved.Built(rec, module, res.Version, refs, nil); err != nil {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded %s either: %w", module, fp, res.Version, when, err)
|
||||
}
|
||||
b.logf("%s at %s: the digests of %s were not recorded at its build's end: recorded %s from the done key", module, fp, res.Version, when)
|
||||
return res.Version, nil
|
||||
}
|
||||
|
||||
// PackageNamed looks at a module again when the indexer's packages state names a name for one of its packages
|
||||
// (versions.NamedModule): a done trunk build that left the package unnumbered, its name not known at its claim, is
|
||||
// numbered then with the done build's digests, and version.numbered says them for its build module's late publish.
|
||||
@@ -199,8 +258,8 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
|
||||
}
|
||||
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
|
||||
for _, t := range plan.Tasks {
|
||||
if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage {
|
||||
src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796)
|
||||
if t.Kind == catalogue.ArtifactUpstream {
|
||||
src.Upstream = append(src.Upstream, module+":"+t.Artifact) // copied by its pin, never awaited (H-A-486)
|
||||
}
|
||||
}
|
||||
res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs, src)
|
||||
|
||||
@@ -32,6 +32,8 @@ type fakeReservations struct {
|
||||
next string // the version Reserve answers when set, else the series' .5
|
||||
fail error // the reservation cannot be made (the catalog's versions cannot be read)
|
||||
tookBy map[string]versions.ReservedVersion // by fingerprint, the reservation that took its series over (H-A-244)
|
||||
held map[string]versions.ReservedVersion // by fingerprint, the reservation ReservedFor answers as it stands (H-A-492)
|
||||
lost error // the record of the digests fails, as a store refusing it would
|
||||
}
|
||||
|
||||
func (f *fakeReservations) TakenOver(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
@@ -46,6 +48,9 @@ func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, e
|
||||
func (f *fakeReservations) Filled() bool { return f.filled }
|
||||
|
||||
func (f *fakeReservations) ReservedFor(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
if v, ok := f.held[fp]; ok {
|
||||
return v, true, nil
|
||||
}
|
||||
for _, s := range f.sources {
|
||||
if s.Fingerprint == fp {
|
||||
return versions.ReservedVersion{Version: "held"}, true, nil
|
||||
@@ -65,7 +70,13 @@ func (f *fakeReservations) Reserve(_ context.Context, module, series string, art
|
||||
return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
func (f *fakeReservations) Built(ctx context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err // the state's write runs under ctx: a cancelled one records nothing
|
||||
}
|
||||
if f.lost != nil {
|
||||
return f.lost
|
||||
}
|
||||
f.deps = deps
|
||||
f.built = map[string]string{version: ""}
|
||||
for ref, d := range digests {
|
||||
@@ -118,6 +129,13 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under the reserved version", res.built)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved for the record and named as upstream, so its publish is never awaited.
|
||||
up := &fakeReservations{filled: true}
|
||||
plan := Plan{Series: "0.1", Tasks: []Task{{Artifact: "server", Kind: catalogue.ArtifactUpstream}, {Artifact: "tools", Kind: catalogue.ArtifactBundle}}}
|
||||
if _, ok := (&Builder{Reserved: up}).reserve(context.Background(), "nats", meshcatalog.ModulesByModuleValue{Fp2: "f"}, plan, Origin{}, false); !ok ||
|
||||
!slices.Equal(up.reserved[0], []string{"nats:server", "nats:tools"}) || !slices.Equal(up.sources[0].Upstream, []string{"nats:server"}) {
|
||||
t.Errorf("an upstream artifact reserved %v from %+v, want both refs reserved and nats:server named upstream", up.reserved, up.sources)
|
||||
}
|
||||
r.consider(t, "modules/m")
|
||||
if len(res.reserved) != 1 {
|
||||
t.Errorf("a done key reserved again: %v", res.reserved)
|
||||
@@ -146,6 +164,97 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound: design 47 H2 H-A-492 — the record of a reservation's digests after a
|
||||
// build ends runs under the builder's own bound, so a build context cancelled at its end still records; a done build whose
|
||||
// digests are not recorded is recorded at the builder's next look, said.
|
||||
func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) {
|
||||
manifest := catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
t.Run("a build's context cancelled at its end still records", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
r.b.Ended = func(context.Context, string) { cancel() } // a takeover or the builder's end, once the key is written
|
||||
r.b.Module(ctx, "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
||||
t.Fatalf("build = %+v, want done", v)
|
||||
}
|
||||
if res.built["m:tool"] != digest {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under 0.1.5 though the build's context ended", res.built)
|
||||
}
|
||||
})
|
||||
t.Run("the record's bound is the setting record-bound-seconds, never the build's", func(t *testing.T) {
|
||||
build, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, c := range []struct {
|
||||
bound, want time.Duration
|
||||
}{{0, DefaultRecordBound}, {5 * time.Second, 5 * time.Second}} {
|
||||
rec, stop := (&Builder{RecordBound: c.bound}).recordContext(build)
|
||||
deadline, has := rec.Deadline()
|
||||
stop()
|
||||
if left := time.Until(deadline); !has || left > c.want || left < c.want-time.Second {
|
||||
t.Errorf("RecordBound %v: the record's context has deadline %v (%v left, %v), want %v left of its own", c.bound, deadline, left, has, c.want)
|
||||
}
|
||||
}
|
||||
rec, stop := (&Builder{}).recordContext(build)
|
||||
defer stop()
|
||||
if rec.Err() != nil {
|
||||
t.Errorf("the record's context ended with the build's: %v", rec.Err())
|
||||
}
|
||||
})
|
||||
t.Run("a build key that cannot be read is said, not taken silently", func(t *testing.T) {
|
||||
var journal []string
|
||||
b := &Builder{Claimer: Claimer{Store: unreadable{newMem()}}, Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }}
|
||||
if d := b.doneDigests(context.Background(), "m", fp1); d != nil {
|
||||
t.Errorf("an unreadable key answered digests %v", d)
|
||||
}
|
||||
if !slices.ContainsFunc(journal, func(l string) bool {
|
||||
return strings.Contains(l, "cannot be read") && strings.Contains(l, "bus timeout")
|
||||
}) {
|
||||
t.Errorf("journal %q, want a line saying the build key cannot be read, with why", journal)
|
||||
}
|
||||
})
|
||||
t.Run("a done key whose digests are not recorded is recorded at the next look", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true, lost: errors.New("context canceled")}
|
||||
r.b.Reserved = res
|
||||
var journal []string
|
||||
r.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
|
||||
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone || res.built != nil {
|
||||
t.Fatalf("build %s recorded %v, want done and nothing recorded", v.State, res.built)
|
||||
}
|
||||
res.lost = nil
|
||||
res.held = map[string]versions.ReservedVersion{fp1: {Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": ""}}}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v at the next look, want the done key's digest of m:tool under 0.1.5", res.built)
|
||||
}
|
||||
if !slices.ContainsFunc(journal, func(l string) bool {
|
||||
return strings.Contains(l, "0.1.5 were not recorded") && strings.Contains(l, "recorded at a look")
|
||||
}) {
|
||||
t.Errorf("journal %q, want a line saying 0.1.5's digests were recorded at a look", journal)
|
||||
}
|
||||
// Recorded once: a reservation whose build ended recorded is left.
|
||||
res.built = nil
|
||||
res.held[fp1] = versions.ReservedVersion{Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": digest}, BuiltAt: time.Unix(1, 0)}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built != nil {
|
||||
t.Errorf("a recorded reservation was recorded again: %v", res.built)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// unreadable is a builds bucket whose reads fail, as one on a bus that does not answer.
|
||||
type unreadable struct{ *memStore }
|
||||
|
||||
func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) {
|
||||
return nil, 0, false, errors.New("bus timeout")
|
||||
}
|
||||
|
||||
// TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through
|
||||
// reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why.
|
||||
func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) {
|
||||
@@ -236,7 +345,7 @@ func TestAReservedBuildKeepsWhatItsStepResolvedAndWasBuiltWith(t *testing.T) {
|
||||
|
||||
// TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion: design 47 G §370 — a module whose one artifact is
|
||||
// a Go package (mesh-wireguard's go-overlay) is planned on the Go build seat, its ask carries the reserved version and names
|
||||
// the seat go-package-registry the package is published and tagged through, and the build is done; a build reserving none
|
||||
// the seat go-package-registry the package is published through, and the build is done; a build reserving none
|
||||
// names no seat.
|
||||
func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *testing.T) {
|
||||
goPackage := catalogue.Artifact{Name: "go-overlay", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}}
|
||||
@@ -265,10 +374,6 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t
|
||||
len(asked[0].Build.Artifacts) != 1 || asked[0].Build.Artifacts[0] != (buildstep.AskArtifact{Artifact: "go-overlay", Seat: c.seat}) {
|
||||
t.Fatalf("asks %+v, want one Go ask with version %q naming go-overlay on seat %q", asked, c.version, c.seat)
|
||||
}
|
||||
// Design 47 H2 §796: the reservation names the Go package's artifact, whose module path names its tag's directory.
|
||||
if c.version != "" && (len(res.sources) != 1 || !res.sources[0].GoPackage || res.sources[0].GoArtifact != "go-overlay") {
|
||||
t.Errorf("reserved from %+v, want one source naming the Go package go-overlay", res.sources)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +107,9 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun,
|
||||
}
|
||||
}
|
||||
// Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture
|
||||
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded.
|
||||
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded. Once the picture
|
||||
// heard a newer entry of the module on the first branch, H-A-489 overrides H-A-205: the older fingerprint is refused
|
||||
// superseded (buildEntry), never built or reserved.
|
||||
b.mu.Lock()
|
||||
for _, e := range run.Entries {
|
||||
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
|
||||
|
||||
@@ -127,6 +127,9 @@ type Builder struct {
|
||||
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished)
|
||||
// gives a pin (H-A-192).
|
||||
PackageWait time.Duration
|
||||
// RecordBound bounds the record of a reservation's digests after a build ends, its setting record-bound-seconds (design 47
|
||||
// H2 H-A-492): the builder's own bound, never the build's context; DefaultRecordBound when 0.
|
||||
RecordBound time.Duration
|
||||
// PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first
|
||||
// (H-A-194): read once before a build waits for that package. Nil reads none.
|
||||
PublishedVersions func(ctx context.Context, path string) ([]string, error)
|
||||
@@ -177,6 +180,8 @@ type Builder struct {
|
||||
// publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished):
|
||||
// a package with no pin waits for one (H-A-192).
|
||||
publishedPins map[string]map[string]string
|
||||
// supersededSaid are the build keys refused as superseded (H-A-489), so each is said once.
|
||||
supersededSaid sync.Map
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -310,11 +315,49 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
|
||||
if superseded != nil {
|
||||
// A newer fingerprint of the module: the build of the older one in flight is taken out, and its asks are withdrawn
|
||||
// from the build seats (ADR 0664 §4).
|
||||
superseded(buildrun.Withdrawn{Why: "the build of " + key + " at " + old.Fp2 + " was superseded by the fingerprint " + m.Fp2})
|
||||
superseded(buildrun.Withdrawn{Why: supersededWhy(key, old.Fp2, m.Fp2)})
|
||||
}
|
||||
b.consider(ctx, key, m)
|
||||
}
|
||||
|
||||
// supersededWhy is the words of a build withdrawn because a newer fingerprint of its module superseded it.
|
||||
func supersededWhy(module, fp, newer string) string {
|
||||
return "the build of " + module + " at " + fp + " was superseded by the fingerprint " + newer
|
||||
}
|
||||
|
||||
// withdrawnPrefix opens the error of a key whose build was withdrawn (H-A-225, ADR 0664 §4). It is final for its fingerprint
|
||||
// (H-A-489): no look builds it again; a person's build again does.
|
||||
const withdrawnPrefix = "withdrawn: "
|
||||
|
||||
// supersededBy is the fingerprint that supersedes fp, a fingerprint of module whose build key is key, when fp is of its
|
||||
// registration's first watched branch and is no longer the newest entry heard on that branch (H-A-489); "" otherwise. A key no
|
||||
// entry named is of the first branch, as originOf has it.
|
||||
func (b *Builder) supersededBy(module, key, fp string) string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
o, ok := b.origin[key]
|
||||
if !ok {
|
||||
o = Origin{First: true}
|
||||
}
|
||||
if !o.First {
|
||||
return ""
|
||||
}
|
||||
newest, heard := b.heard[o.Branch+"\x00"+module]
|
||||
if !heard || newest.Fp2 == "" || newest.Fp2 == fp {
|
||||
return ""
|
||||
}
|
||||
return newest.Fp2
|
||||
}
|
||||
|
||||
// saySuperseded says, once per key, that the build of fp is refused as superseded by newer (H-A-489), and gives the words.
|
||||
func (b *Builder) saySuperseded(module, key, fp, newer string) string {
|
||||
why := "is not built: superseded by " + newer + ", the newest entry heard on its first branch"
|
||||
if _, said := b.supersededSaid.LoadOrStore(key, true); !said {
|
||||
b.logf("%s at %s: %s", module, fp, why)
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
|
||||
// directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is
|
||||
// built against, its packages and needs, and the named paths of its artifacts when every artifact names them.
|
||||
@@ -637,6 +680,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
whyEnded = fmt.Sprintf(format, args...)
|
||||
b.logf("%s at %s: %s", module, m.Fp2, whyEnded)
|
||||
}
|
||||
// A fingerprint no longer the newest entry heard on its first branch is neither reserved for nor built (H-A-489): two
|
||||
// fingerprints of one module never take the series from each other.
|
||||
if newer := b.supersededBy(module, key, m.Fp2); newer != "" {
|
||||
whyEnded = b.saySuperseded(module, key, m.Fp2, newer)
|
||||
return
|
||||
}
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
var noModule *NoBuildModule
|
||||
if errors.As(planErr, &noModule) {
|
||||
@@ -680,6 +729,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
|
||||
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
|
||||
if trunk && b.originOf(key).First && planErr == nil && !nothing && !reserving {
|
||||
// A done build whose digests its end did not record is recorded at the next look (design 47 H2 H-A-492).
|
||||
if _, err := b.RecordAgain(ctx, module, m.Fp2, "at a look"); err != nil {
|
||||
b.logf("%v", err)
|
||||
}
|
||||
}
|
||||
if reserving && b.waitFill(module, m) {
|
||||
end("is not built yet: wait-held, the catalog has not said versions-filled")
|
||||
return
|
||||
@@ -727,7 +782,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
}
|
||||
var reserved versions.ReservedVersion
|
||||
hasReserved := false
|
||||
hasReserved, supersededNow := false, ""
|
||||
if rebuild && cl.Build.State == builds.BuildBuilding {
|
||||
// Reserved as A-490b amended by H-A-225 has it: this fingerprint's standing reservation is kept while it is unreleased
|
||||
// and above the highest the catalog's versions answers, else the next above that highest is taken (A-455); the build's
|
||||
@@ -737,10 +792,16 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
reserving = true
|
||||
} else if reserving && cl.Build.State != builds.BuildDone {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
// Looked at again just before it reserves (H-A-489): a newer entry heard since the claim is not taken over.
|
||||
if supersededNow = b.supersededBy(module, key, m.Fp2); supersededNow == "" {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
}
|
||||
}
|
||||
var result builds.BuilderBuild
|
||||
if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
if supersededNow != "" {
|
||||
// Ended as the newer entry's take-in withdraws a build in flight: final for this fingerprint.
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + supersededWhy(module, m.Fp2, supersededNow)})}
|
||||
} else if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
// A trunk build that reserves is never built without its version: it would end done and publish nothing, and its key would
|
||||
// stay done, never built again (A-490). The key fails with why and is taken again at the next look or by builder_build_again.
|
||||
result = failed("no version is reserved for this build: the builder's journal says why, and it is built again once one can be")
|
||||
@@ -765,7 +826,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
var w buildrun.Withdrawn
|
||||
if ctx.Err() == nil && errors.As(context.Cause(run), &w) {
|
||||
// Taken out, not stopped: the key says so, and the asks are withdrawn (the asker dropped them).
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{"withdrawn: " + w.Why})}
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + w.Why})}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
@@ -807,8 +868,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
for _, a := range result.Artifacts {
|
||||
digests[module+":"+a.Name] = a.Digest
|
||||
}
|
||||
if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err)
|
||||
// Under the builder's own bound, never the build's context: a takeover or an end that cancels it no longer leaves a passed
|
||||
// build unrecorded, so unreleased (design 47 H2 H-A-492).
|
||||
rec, stop := b.recordContext(ctx)
|
||||
err := b.Reserved.Built(rec, module, reserved.Version, digests, b.takeDependencies(key))
|
||||
stop()
|
||||
if err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v; recorded again at the builder's next look or start", cl.Key, reserved.Version, err)
|
||||
}
|
||||
}
|
||||
if result.State == builds.BuildFailed {
|
||||
|
||||
@@ -333,8 +333,9 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
asked, release := make(chan struct{}), make(chan struct{})
|
||||
answer := r.answer
|
||||
var once sync.Once
|
||||
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
||||
close(asked)
|
||||
once.Do(func() { close(asked) })
|
||||
<-release
|
||||
return answer(a)
|
||||
}
|
||||
@@ -356,6 +357,85 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "withdrawn: its reservation 0.1.3 was taken over") {
|
||||
t.Fatalf("the key = %+v, want failed withdrawn naming the take-over", v)
|
||||
}
|
||||
// H-A-489: withdrawn is final for the fingerprint: a later look builds it not again; a person's build again does.
|
||||
n, rev := len(r.asks.Asks()), r.st.revs[r.key]
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); len(r.asks.Asks()) != n || r.st.revs[r.key] != rev || !strings.HasPrefix(v.Error, "withdrawn: ") {
|
||||
t.Errorf("a later look built the withdrawn key again: %d asks (was %d), the key %+v", len(r.asks.Asks()), n, v)
|
||||
}
|
||||
if cl, err := r.b.Claimer.Claim(context.Background(), "m", fp1, mdigest('a'), ClaimOptions{ByHand: true}); err != nil || cl == nil {
|
||||
t.Errorf("a person's build again of the withdrawn key was not claimed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-489 (review round 1): a newer entry of the first branch heard between the older build's claim and its reservation is
|
||||
// not taken over: the older key ends failed withdrawn with no reservation, and the newer reserves.
|
||||
func TestANewerEntryHeardBeforeTheReservationIsNotTakenOver(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
newer, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
Fp2: fp2, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.st.before = func() { r.b.Module(context.Background(), "m", newer) } // at the older key's claim
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, "withdrawn: ") || !strings.Contains(v.Error, "superseded by the fingerprint "+fp2) {
|
||||
t.Errorf("the older key = %+v, want failed withdrawn, superseded by %s", v, fp2)
|
||||
}
|
||||
if len(res.sources) != 1 || res.sources[0].Fingerprint != fp2 {
|
||||
t.Errorf("reserved %+v, want one reservation, the newer's", res.sources)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-489: a fingerprint of the first branch that is no longer its newest entry heard is neither reserved for nor built, and
|
||||
// said once; the again loop that looks at it later builds nothing.
|
||||
func TestAnOlderFingerprintAskedAfterANewerEntryIsRefused(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
r.b.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
r.bm.fails["ace/build_go_build"] = "go build of tool: boom"
|
||||
older := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: fp1, Manifest: mdigest('a'),
|
||||
Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
delete(r.bm.fails, "ace/build_go_build")
|
||||
newer := older
|
||||
newer.Fp2 = fp2
|
||||
data, err := meshcatalogclient.EncodeModule(newer)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
r.b.Wait()
|
||||
rev := r.st.revs[r.key]
|
||||
for range 2 {
|
||||
r.b.consider(context.Background(), "m", older) // as consider's again loop or a step's look takes it
|
||||
r.b.Wait()
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || r.st.revs[r.key] != rev {
|
||||
t.Errorf("the older key = %+v at revision %d (was %d), want it left as it failed", v, r.st.revs[r.key], rev)
|
||||
}
|
||||
k2, _ := builds.BuilderBuildKey("m", fp2)
|
||||
if v := valueOf(t, r.st, k2); v.State != builds.BuildDone {
|
||||
t.Errorf("the newer key = %+v, want done", v)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range said {
|
||||
if strings.Contains(l, fp1+": is not built: superseded by "+fp2) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("superseded said %d times, want once: %q", n, said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key
|
||||
|
||||
@@ -142,6 +142,10 @@ func (b *Builder) buildEntry(ctx context.Context, pipeline string, e meshbuilds.
|
||||
}
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, failedWords, "existing build")
|
||||
}
|
||||
if newer := b.supersededBy(entry.Module, key, entry.Fp); newer != "" {
|
||||
// Refused, not waited on (H-A-489): no look builds a superseded fingerprint, so its end would never come.
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s %s", entry.Module, entry.Fp, b.saySuperseded(entry.Module, key, entry.Fp, newer)), "refused")
|
||||
}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
|
||||
b.consider(ctx, entry.Module, m)
|
||||
@@ -232,7 +236,7 @@ func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed str
|
||||
switch {
|
||||
case v.State == builds.BuildDone && !Unpublished(v, b.Seats):
|
||||
return true, "", true
|
||||
case v.State == builds.BuildFailed && builds.FailedUntilChanged(v.Error):
|
||||
case v.State == builds.BuildFailed && (builds.FailedUntilChanged(v.Error) || strings.HasPrefix(v.Error, withdrawnPrefix)):
|
||||
return false, key + ": " + v.Error, true
|
||||
}
|
||||
return false, "", false
|
||||
|
||||
@@ -74,6 +74,21 @@ func TestABuildStepAnswersItsEnd(t *testing.T) {
|
||||
if end := ask(entries...); end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Step != meshbuilds.StepEndedStepBuild || len(end.Entries) != 1 || end.Entries[0].Commit != "c1" || end.Pipeline != "main.1.m" {
|
||||
t.Errorf("a done build: %+v", end)
|
||||
}
|
||||
// H-A-489 (review round 1): a withdrawn key whose fingerprint is still the newest entry is answered failed at once, naming
|
||||
// withdrawn: no look builds it again, so its end is never waited for.
|
||||
rw := newRig(t, catalogue.Artifact{Name: "files", Kind: "archive"})
|
||||
rw.b.StepEntry = r.b.StepEntry
|
||||
withdrawn, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Manifest: mdigest('a'), Error: withdrawnPrefix + "its reservation 0.1.3 was taken over"})
|
||||
if _, err := rw.st.Create(t.Context(), rw.key, withdrawn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rw.b.Module(t.Context(), "m", moduleValue(t, "modules/m")) // the newest entry heard is fp1
|
||||
rw.b.Wait()
|
||||
bounded, cancel := context.WithTimeout(t.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
if end := rw.b.BuildStep(bounded, "ask1", meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries}); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "withdrawn: ") {
|
||||
t.Errorf("a withdrawn key asked: %+v, want failed at once naming withdrawn", end)
|
||||
}
|
||||
}
|
||||
|
||||
// A step ask waits for the end of its build without a timer: the log entry of that end wakes it (design 47 G A-97).
|
||||
@@ -434,9 +449,10 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved
|
||||
// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs;
|
||||
// a kept revision naming another fingerprint fails the step, naming it.
|
||||
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, while the older pipeline
|
||||
// asks the build of revision 45. Its kept catalog revision is read by number and the build runs while the picture never heard
|
||||
// a newer entry of m (a restart); once it heard one on the first branch, H-A-489 overrides H-A-205 and the older ask is refused
|
||||
// superseded. A kept revision naming another fingerprint fails the step, naming it.
|
||||
func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
var held atomic.Int64
|
||||
held.Store(46)
|
||||
@@ -460,6 +476,24 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 {
|
||||
t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources)
|
||||
}
|
||||
// H-A-489: once the picture heard the newer entry of m on the first branch, the older ask is refused superseded and
|
||||
// nothing is built or reserved for it; the newer entry's build keeps the series.
|
||||
rn, resn, entryn := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
rn.b.RevisionWait = time.Nanosecond
|
||||
rn.b.FirstBranch = r.b.FirstBranch
|
||||
rn.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
return 45, map[string]string{"m": fp1}, nil
|
||||
}
|
||||
entryn.Fp = fp2
|
||||
rn.b.Entry(context.Background(), Origin{Branch: "main", Source: 46, First: true}, entryn)
|
||||
rn.b.Wait()
|
||||
if endn := rn.b.BuildStep(t.Context(), "a1", run); endn.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(endn.Said, "superseded by "+fp2) {
|
||||
t.Errorf("the older ask after the newer entry: end = %+v; want failed, superseded by %s", endn, fp2)
|
||||
}
|
||||
rn.b.Wait()
|
||||
if _, built := rn.st.vals[rn.key]; built || len(resn.sources) != 1 || resn.sources[0].Fingerprint != fp2 {
|
||||
t.Errorf("built %v, reserved %+v; want nothing built for the older, one reservation, the newer's", built, resn.sources)
|
||||
}
|
||||
r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
r2.b.RevisionWait = time.Nanosecond
|
||||
r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
|
||||
@@ -10,10 +10,8 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -42,8 +40,8 @@ const (
|
||||
// VersionHeldElsewhere is the finding of a number a registry seat already holds, skipped at reserving or moved from (design 47
|
||||
// H H-A-275), scope module.
|
||||
VersionHeldElsewhere = "version-held-elsewhere"
|
||||
// VersionTaggedElsewhere is the finding of a number whose git tag stands at another commit, skipped at reserving (design 47
|
||||
// H2 §796), scope module; cleared at the module's next version-released.
|
||||
// VersionTaggedElsewhere is the finding of a number whose git tag stood at another commit, which a reserve skipped before no
|
||||
// tag was read (design 47 H2 H-A-494); one raised before is cleared at the module's next version-released.
|
||||
VersionTaggedElsewhere = "version-tagged-elsewhere"
|
||||
)
|
||||
|
||||
@@ -55,6 +53,10 @@ type ReservedVersion struct {
|
||||
Version string `json:"version"` // MAJOR.MINOR.PATCH
|
||||
// Artifacts are the module's artifacts at the version: the digest once built, empty before.
|
||||
Artifacts map[string]string `json:"artifacts"`
|
||||
// Upstream are the artifacts of kind upstream, by reference (design 47 H2 H-A-486): kept in Artifacts with their digest for the
|
||||
// record, never awaited, as design 51 copies an upstream image by its pinned digest and no registry seat says it published at
|
||||
// a version. A reservation written before carries none.
|
||||
Upstream []string `json:"upstream,omitempty"`
|
||||
// BuiltAt is when the build ended with every digest known; the publish's bound counts from it.
|
||||
BuiltAt time.Time `json:"built-at,omitzero"`
|
||||
// Published are the artifacts a registry seat said published at this version with the built digest.
|
||||
@@ -63,8 +65,8 @@ type ReservedVersion struct {
|
||||
// was kept.
|
||||
PublishedAt map[string]time.Time `json:"published-at,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go
|
||||
// package's, GoTagOf): empty for a module with no Go package (design 47 G §370), or whose Go package's directory is not known.
|
||||
// Repository and Commit are what the build was made from. Tag is kept for a reservation written before design 47 H2 H-A-494:
|
||||
// no git tag is written at a release any more, and a new reservation names none.
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
@@ -110,22 +112,19 @@ type KeptPublished struct {
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
|
||||
// written at, and whether the module has a Go package, whose version is tagged.
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit, kept on the reservation.
|
||||
type Source struct {
|
||||
Repository, Commit string
|
||||
// Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76).
|
||||
Fingerprint string
|
||||
GoPackage bool
|
||||
// GoArtifact is the artifact that is the module's Go package, whose module path names the directory its tag is written in
|
||||
// (design 47 H2 §796).
|
||||
GoArtifact string
|
||||
// Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of
|
||||
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
|
||||
Branch string
|
||||
Revision int64
|
||||
// CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did.
|
||||
CatalogRevision int64
|
||||
// Upstream are the artifacts of kind upstream among those reserved, by reference (H-A-486), kept on the reservation.
|
||||
Upstream []string
|
||||
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
|
||||
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
|
||||
Rebuild bool
|
||||
@@ -171,92 +170,6 @@ type Reserver struct {
|
||||
HeldNumbers func(ctx context.Context, module, fingerprint string, refs []string) ([]string, error)
|
||||
// HeldElsewhere is told the numbers of module a reserve skipped because a registry holds them (H-A-275); nil is told none.
|
||||
HeldElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
// Names answers a Go package's module path, which names the directory its tag is written in (design 47 H2 §796); nil, or a
|
||||
// name not known, tags the version nowhere, said.
|
||||
Names PackageNames
|
||||
// Tags reads repository's tags starting with prefix, each with the commit it names, through the seat git's tags (design 47
|
||||
// H2 §796); away says the read met a transport fault or the error kind away, which lets the reservation go on, said. Nil
|
||||
// reads none.
|
||||
Tags func(ctx context.Context, repository, prefix string) (tags []git.Tag, away bool, err error)
|
||||
// TaggedElsewhere is told the numbers of module a reserve skipped because their tag stands at another commit (§796); nil
|
||||
// is told none.
|
||||
TaggedElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
|
||||
tagsMu sync.Mutex
|
||||
// tagsRead are the tags last read of each module, by series patch to the commit, with the catalog revision they were read
|
||||
// under: a module's tags are read once per catalog revision it reserves under (§796).
|
||||
tagsRead map[string]tagsAt
|
||||
}
|
||||
|
||||
// tagsAt are a module's tags of one series read under one catalog revision: patch to the commit its tag names.
|
||||
type tagsAt struct {
|
||||
revision, prefix string
|
||||
commits map[int]string
|
||||
}
|
||||
|
||||
// goDirectory is the directory of module's Go package in its repository (§796), from its module path; ok is false, said, when
|
||||
// no name or no directory of the repository is known.
|
||||
func (r *Reserver) goDirectory(ctx context.Context, module string, src Source) (string, bool) {
|
||||
if r.Names == nil {
|
||||
return "", false
|
||||
}
|
||||
path, known, err := r.Names(ctx, module, src.GoArtifact)
|
||||
switch {
|
||||
case err != nil:
|
||||
r.said(fmt.Errorf("%s: its Go package's name cannot be read, so its version is tagged nowhere: %w", module, err))
|
||||
return "", false
|
||||
case !known:
|
||||
r.said(fmt.Errorf("%s: no name of its Go package %s is known yet, so its version is tagged nowhere", module, src.GoArtifact))
|
||||
return "", false
|
||||
}
|
||||
dir, ok := GoDirectory(path, src.Repository)
|
||||
if !ok {
|
||||
r.said(fmt.Errorf("%s: its Go package %s is not a module path of the repository %s, so its version is tagged nowhere", module, path, src.Repository))
|
||||
}
|
||||
return dir, ok
|
||||
}
|
||||
|
||||
// taggedAt are the patches of series whose tag `<directory>/v<series>.<patch>` stands, each with its commit, read once per
|
||||
// catalog revision the module reserves under (§796). A read that meets a transport fault or away answers none, said, and the
|
||||
// reservation goes on; any other failure refuses the reservation, naming it.
|
||||
func (r *Reserver) taggedAt(ctx context.Context, module, series, dir string, src Source) (map[int]string, error) {
|
||||
if r.Tags == nil {
|
||||
return nil, nil
|
||||
}
|
||||
prefix := TagPrefix(dir) + series + "."
|
||||
revision := ""
|
||||
if src.CatalogRevision > 0 {
|
||||
revision = fmt.Sprintf("%s.%d", src.Branch, src.CatalogRevision)
|
||||
r.tagsMu.Lock()
|
||||
got, ok := r.tagsRead[module]
|
||||
r.tagsMu.Unlock()
|
||||
if ok && got.revision == revision && got.prefix == prefix {
|
||||
return got.commits, nil
|
||||
}
|
||||
}
|
||||
tags, away, err := r.Tags(ctx, src.Repository, prefix)
|
||||
if err != nil && away {
|
||||
r.said(fmt.Errorf("%s: its tags %s* cannot be read (%w), so the reservation goes on without them", module, prefix, err))
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the tags %s* of %s cannot be read: %w", prefix, module, err)
|
||||
}
|
||||
commits := map[int]string{}
|
||||
for _, t := range tags {
|
||||
if s, p, err := SplitVersion(strings.TrimPrefix(t.Tag, TagPrefix(dir))); err == nil && s == series && strings.HasPrefix(t.Tag, prefix) {
|
||||
commits[p] = t.Commit
|
||||
}
|
||||
}
|
||||
if revision != "" {
|
||||
r.tagsMu.Lock()
|
||||
if r.tagsRead == nil {
|
||||
r.tagsRead = map[string]tagsAt{}
|
||||
}
|
||||
r.tagsRead[module] = tagsAt{revision: revision, prefix: prefix, commits: commits}
|
||||
r.tagsMu.Unlock()
|
||||
}
|
||||
return commits, nil
|
||||
}
|
||||
|
||||
// heldNumbers are the patches of series a registry holds of module's artifacts (H-A-275); a read that fails is said, and none.
|
||||
@@ -277,19 +190,15 @@ func (r *Reserver) heldNumbers(ctx context.Context, module, series, fingerprint
|
||||
return out
|
||||
}
|
||||
|
||||
// nextFree is the first patch after after that no registry holds and whose tag stands at no other commit than commit, with the
|
||||
// held numbers it skipped (H-A-275) and those tagged elsewhere (design 47 H2 §796).
|
||||
func nextFree(series string, after int, held map[int]bool, tagged map[int]string, commit string) (int, []string, []string) {
|
||||
var skipped, elsewhere []string
|
||||
// nextFree is the first patch after after that no registry holds, with the held numbers it skipped (H-A-275). No git tag is
|
||||
// read: a series is numbered from the catalog's versions and the registries' alone (design 47 H2 H-A-494).
|
||||
func nextFree(series string, after int, held map[int]bool) (int, []string) {
|
||||
var skipped []string
|
||||
for p := after + 1; ; p++ {
|
||||
switch c, ok := tagged[p]; {
|
||||
case held[p]:
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
case ok && c != commit:
|
||||
elsewhere = append(elsewhere, fmt.Sprintf("%s.%d (tagged at %s)", series, p, c))
|
||||
default:
|
||||
return p, skipped, elsewhere
|
||||
if !held[p] {
|
||||
return p, skipped
|
||||
}
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,15 +251,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
}
|
||||
key := ReservedKey(module, series)
|
||||
held := r.heldNumbers(ctx, module, series, src.Fingerprint, artifacts)
|
||||
dir, tagged := "", false
|
||||
var tags map[int]string
|
||||
if src.GoPackage {
|
||||
if dir, tagged = r.goDirectory(ctx, module, src); tagged {
|
||||
if tags, err = r.taggedAt(ctx, module, series, dir, src); err != nil {
|
||||
return ReservedVersion{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
for {
|
||||
cur, rev, found, err := r.get(ctx, key)
|
||||
if err != nil {
|
||||
@@ -368,15 +268,13 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
next = p
|
||||
}
|
||||
}
|
||||
patch, skipped, elsewhere := nextFree(series, next, held, tags, src.Commit)
|
||||
patch, skipped := nextFree(series, next, held)
|
||||
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, patch), Artifacts: map[string]string{}, At: r.now(),
|
||||
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
|
||||
if tagged {
|
||||
out.Tag = GoTagOf(dir, out.Version)
|
||||
}
|
||||
for _, a := range artifacts {
|
||||
out.Artifacts[a] = ""
|
||||
}
|
||||
out.Upstream = slices.Clone(src.Upstream)
|
||||
if found {
|
||||
out.Withdrawn = maps.Clone(cur.Withdrawn)
|
||||
if cur.Fingerprint != src.Fingerprint {
|
||||
@@ -410,9 +308,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
if len(skipped) > 0 && r.HeldElsewhere != nil {
|
||||
r.HeldElsewhere(ctx, module, skipped)
|
||||
}
|
||||
if len(elsewhere) > 0 && r.TaggedElsewhere != nil {
|
||||
r.TaggedElsewhere(ctx, module, elsewhere)
|
||||
}
|
||||
if found && r.Took != nil {
|
||||
r.Took(ctx, cur, out)
|
||||
}
|
||||
@@ -1134,11 +1029,11 @@ const (
|
||||
ReservedVerb = meshbuilds.VerbReserved
|
||||
)
|
||||
|
||||
// Missing are the artifacts of v no registry seat said published, sorted.
|
||||
// Missing are the artifacts of v no registry seat said published, sorted; an upstream one is never awaited (H-A-486).
|
||||
func (v ReservedVersion) Missing() []string {
|
||||
var out []string
|
||||
for a := range v.Artifacts {
|
||||
if !v.Published[a] {
|
||||
if !v.Published[a] && !slices.Contains(v.Upstream, a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
)
|
||||
|
||||
@@ -171,6 +169,23 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
if got := cur.Missing(); !reflect.DeepEqual(got, []string{"m:b"}) {
|
||||
t.Errorf("missing = %v, want m:b", got)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved and recorded with its digest, but never awaited: once the others are published
|
||||
// nothing is missing.
|
||||
up, err := r.Reserve(ctx, "u", "0.1", []string{"u:server", "u:tools"}, Source{Upstream: []string{"u:server"}})
|
||||
if err != nil || !reflect.DeepEqual(up.Upstream, []string{"u:server"}) {
|
||||
t.Fatalf("reserved %+v (%v), want u:server kept as upstream", up, err)
|
||||
}
|
||||
for _, a := range []string{"u:server", "u:tools"} {
|
||||
if _, err := r.Built(ctx, "u", up.Version, a, "sha256:"+a, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if marked, _, err := r.Published(ctx, "u-tools", up.Version, "sha256:u:tools"); err != nil || len(marked) != 1 {
|
||||
t.Fatalf("the published of u:tools marked %v (%v)", marked, err)
|
||||
}
|
||||
if cur, _, _ := r.At(ctx, ReservedKey("u", "0.1")); len(cur.Missing()) != 0 || cur.Artifacts["u:server"] != "sha256:u:server" {
|
||||
t.Errorf("missing = %v with artifacts %v, want none missing and the upstream digest kept", cur.Missing(), cur.Artifacts)
|
||||
}
|
||||
if key, ok := ReservedBoundOf(ReservedBoundKey("m", "1.4")); !ok || key != ReservedKey("m", "1.4") || ReservedBoundFor(key) != ReservedBoundKey("m", "1.4") {
|
||||
t.Errorf("the bound key does not name the reservation back: %q %v", key, ok)
|
||||
}
|
||||
@@ -399,7 +414,7 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
return held, readErr
|
||||
},
|
||||
HeldElsewhere: func(_ context.Context, _ string, s []string) { skipped = append(skipped, s...) }}
|
||||
v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true})
|
||||
v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1"})
|
||||
if err != nil || v.Version != "0.1.39" || len(skipped) != 2 {
|
||||
t.Fatalf("reserved %q (%v), skipped %v: want 0.1.39 past the two numbers the registry holds", v.Version, err, skipped)
|
||||
}
|
||||
@@ -413,9 +428,9 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
}
|
||||
held = []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40", "0.2.0"}
|
||||
// The build again of a done key reserves as a rebuild (A-490b): a withdrawn number is never its floor's to reuse.
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true, Rebuild: true})
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1", Rebuild: true})
|
||||
if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != "" {
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (no Go package name known)", again, err)
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (design 47 H2 H-A-494)", again, err)
|
||||
}
|
||||
if at, _ := r.AtVersion(ctx, "0.1.39"); len(at) != 0 {
|
||||
t.Errorf("the index by version still names the withdrawn number: %v", at)
|
||||
@@ -428,50 +443,4 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
if v, err := r.Reserve(ctx, "node-tools", "0.2", []string{"node-tools:program"}, Source{Fingerprint: "fp1"}); err != nil || v.Version != "0.2.1" {
|
||||
t.Errorf("with the registry down reserved %q (%v), want 0.2.1 (what it answered still skipped)", v.Version, err)
|
||||
}
|
||||
// Design 47 H2 §796: a Go package's tags are read before it reserves, once per catalog revision; a number whose tag
|
||||
// `<directory>/v<version>` stands at another commit is skipped and said, one at the build's own commit is not; the version's
|
||||
// tag names the Go module's directory; a read met away lets the reservation go on, any other failure refuses it.
|
||||
r.HeldNumbers = nil
|
||||
var tagReads int
|
||||
var tagsErr error
|
||||
var tagsAway bool
|
||||
var elsewhere []string
|
||||
tags := []git.Tag{{Tag: "modules/nt/v0.3.0", Commit: "c-other"}, {Tag: "modules/nt/v0.3.1", Commit: "c-mine"}, {Tag: "modules/nt/v0.3.2", Commit: "c-other2"}}
|
||||
r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/nt", module == "node-tools" && artifact == "lib", nil
|
||||
}
|
||||
r.Tags = func(_ context.Context, repository, prefix string) ([]git.Tag, bool, error) {
|
||||
tagReads++
|
||||
if repository != "o/r" || prefix != "modules/nt/v0.3." {
|
||||
t.Errorf("tags read of %s %q, want o/r modules/nt/v0.3.", repository, prefix)
|
||||
}
|
||||
return tags, tagsAway, tagsErr
|
||||
}
|
||||
r.TaggedElsewhere = func(_ context.Context, _ string, s []string) { elsewhere = append(elsewhere, s...) }
|
||||
at := Source{Fingerprint: "fp3", GoPackage: true, GoArtifact: "lib", Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 7}
|
||||
v3, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at)
|
||||
if err != nil || v3.Version != "0.3.1" || v3.Tag != "modules/nt/v0.3.1" || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.0") {
|
||||
t.Fatalf("reserved %+v (%v), tagged elsewhere %v: want 0.3.1 (0.3.0 tagged at another commit, 0.3.1 at the build's own), tag modules/nt/v0.3.1", v3, err, elsewhere)
|
||||
}
|
||||
at.Fingerprint, at.Commit, elsewhere = "fp4", "c-new", nil
|
||||
if v4, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v4.Version != "0.3.3" || tagReads != 1 || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.2") {
|
||||
t.Fatalf("reserved %+v (%v) after %d reads, tagged elsewhere %v: want 0.3.3 past 0.3.2's tag, the tags read once for revision main.7", v4, err, tagReads, elsewhere)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsErr, tagsAway = 8, "fp5", errors.New("the forge is away"), true
|
||||
if v5, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v5.Version != "0.3.4" || tagReads != 2 {
|
||||
t.Fatalf("with the tags away reserved %+v (%v) after %d reads, want 0.3.4 and the reservation going on", v5, err, tagReads)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsAway = 9, "fp6", false
|
||||
if _, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err == nil || !strings.Contains(err.Error(), "tags") {
|
||||
t.Errorf("a tags read refused not away reserved anyway (%v), want the reservation refused naming the tags", err)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/novox/mesh-sdk/go", "https://git.example/novox/mesh-sdk.git"); !ok || d != "go" || GoTagOf(d, "0.1.9") != "go/v0.1.9" {
|
||||
t.Errorf("mesh-sdk's Go package: directory %q (%v), want go and the tag go/v0.1.9", d, ok)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/o/r", "o/r"); !ok || d != "" || GoTagOf(d, "1.0.0") != "v1.0.0" {
|
||||
t.Errorf("a root Go module: directory %q (%v), want the root and the tag v1.0.0", d, ok)
|
||||
}
|
||||
if _, ok := GoDirectory("git.example/o/rr/x", "o/r"); ok {
|
||||
t.Error("a module path of another repository was given a directory")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package versions
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// GoTagOf is the git tag a Go package's released version is written as (design 47 H2 §796): `<directory>/v<version>`, the
|
||||
// directory of its Go module in the repository, which the Go toolchain's direct fallback reads; `v<version>` at the root. The
|
||||
// `go/v<version>` tags go from new versions.
|
||||
func GoTagOf(directory, version string) string { return TagPrefix(directory) + version }
|
||||
|
||||
// GoDirectory is the directory of the Go module path modulePath in repository (`<owner>/<name>`, or a clone address naming
|
||||
// it): the part of the path after the repository's own, as the Go toolchain reads a module kept in a subdirectory; empty at
|
||||
// the root. ok is false when the path is not one of that repository.
|
||||
func GoDirectory(modulePath, repository string) (string, bool) {
|
||||
repo := "/" + strings.ToLower(trunk.RepositoryName(repository))
|
||||
lower := strings.ToLower(modulePath)
|
||||
for from := 0; ; {
|
||||
i := strings.Index(lower[from:], repo)
|
||||
if i < 0 || repo == "/" {
|
||||
return "", false
|
||||
}
|
||||
end := from + i + len(repo)
|
||||
if end == len(lower) || lower[end] == '/' {
|
||||
return strings.Trim(modulePath[end:], "/"), true
|
||||
}
|
||||
from = end
|
||||
}
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
// Design 51 *The floor*: the floor's three sources, the registry seats' versions, git.tags and indexer_requirements.
|
||||
// The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3).
|
||||
if want := []string{"seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
|
||||
"seat:git.tag", "seat:git.tags",
|
||||
"seat:git.tags", // the floor's source alone: no tag is written at a release (design 47 H2 H-A-494)
|
||||
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
|
||||
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions",
|
||||
// A-231: the build asks on each build seat's queue delivery (accept.delivery.<asker>).
|
||||
@@ -199,15 +199,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 {
|
||||
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
|
||||
}
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 ||
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 13 ||
|
||||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
|
||||
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
|
||||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime ||
|
||||
// H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default
|
||||
m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime ||
|
||||
m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
// H-A-492: the record of a version's digests runs under the builder's own bound, a setting of 60 s by default
|
||||
m.Config["record-bound-seconds"].Default != float64(60) ||
|
||||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 {
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), record-bound-seconds of 60 (H-A-492), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
}
|
||||
// Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends.
|
||||
if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
@@ -266,7 +268,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("health = %v, want kind unit alone", p["health"])
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
if len(env) != 9 || env["MESH_BUILDER_RECORD_BOUND_FILE"] != "${dir:state}/record-bound-seconds" || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
|
||||
}
|
||||
for k, v := range env {
|
||||
@@ -274,14 +276,16 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("env %s = %v: a setting reaches the process as a file, never its environment (A-71)", k, v)
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "file" && r["path"] == "${dir:state}/toolchain-wait-minutes" && r["content"] == "${config:toolchain-wait-minutes}" {
|
||||
found = true
|
||||
for _, setting := range []string{"toolchain-wait-minutes", "record-bound-seconds"} { // H-A-492: the record's bound too
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "file" && r["path"] == "${dir:state}/"+setting && r["content"] == "${config:"+setting+"}" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no file resource with the content ${config:%s}: A-71", setting)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no file resource with the content ${config:toolchain-wait-minutes}: A-71")
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "settings" {
|
||||
|
||||
@@ -91,7 +91,6 @@
|
||||
"invokes": [
|
||||
"seat:container-image-registry.remove",
|
||||
"seat:container-image-registry.versions",
|
||||
"seat:git.tag",
|
||||
"seat:git.tags",
|
||||
"seat:go-package-registry.remove",
|
||||
"seat:go-package-registry.versions",
|
||||
@@ -183,6 +182,14 @@
|
||||
"trusted": false,
|
||||
"default": 10
|
||||
},
|
||||
"record-bound-seconds": {
|
||||
"type": "number",
|
||||
"description": "seconds the builder gives the record of a version's digests after its build ends, under its own bound and never the build's (design 47 H2 H-A-492).",
|
||||
"minimum": 1,
|
||||
"maximum": 3600,
|
||||
"trusted": false,
|
||||
"default": 60
|
||||
},
|
||||
"ask-taken-bound": {
|
||||
"type": "string",
|
||||
"description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).",
|
||||
@@ -310,6 +317,14 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:package-wait-minutes}"
|
||||
},
|
||||
{
|
||||
"id": "record-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/record-bound-seconds",
|
||||
"owner": "mesh-delivery-builder",
|
||||
"mode": "0600",
|
||||
"content": "${config:record-bound-seconds}"
|
||||
},
|
||||
{
|
||||
"id": "publish-pause",
|
||||
"type": "file",
|
||||
@@ -361,6 +376,7 @@
|
||||
"MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes",
|
||||
"MESH_BUILDER_REVISION_WAIT_FILE": "${dir:state}/revision-wait-minutes",
|
||||
"MESH_BUILDER_PACKAGE_WAIT_FILE": "${dir:state}/package-wait-minutes",
|
||||
"MESH_BUILDER_RECORD_BOUND_FILE": "${dir:state}/record-bound-seconds",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_FILE": "${dir:state}/publish-pause",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE": "${dir:state}/publish-pause-max",
|
||||
"MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries",
|
||||
|
||||
@@ -14,13 +14,13 @@ const Seat = "mesh-pipeline-runner"
|
||||
|
||||
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
|
||||
// a reader of a value of another form says "form N, this reader knows M".
|
||||
const Form = 8
|
||||
const Form = 9
|
||||
|
||||
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
|
||||
// whose fields changed while its forms did not (design 47 H2 §780).
|
||||
const (
|
||||
generatedForms = "8"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionAgainNValue=693580eed498 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
generatedForms = "9"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=582f8ff3b28d PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionAgainNValue=693580eed498 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -948,7 +948,7 @@ const (
|
||||
PipelineRowOutcomeCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// PipelineStarted is a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52).
|
||||
// PipelineStarted is a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52), and again at a re-pin with its kept revisions (H2 H-A-496).
|
||||
type PipelineStarted struct {
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
// the watched branch, or the branch of a run on request
|
||||
@@ -957,6 +957,8 @@ type PipelineStarted struct {
|
||||
Call string `json:"call,omitempty"`
|
||||
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
|
||||
Form int64 `json:"form,omitempty"`
|
||||
// every catalog revision the pipeline's movers are kept at, said again in full at each re-pin, so the indexer pins each while the pipeline runs (design 47 H2 H-A-496 and its addendum)
|
||||
Kept []int64 `json:"kept,omitempty"`
|
||||
// the pipeline's movers
|
||||
Movers []Mover `json:"movers"`
|
||||
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
|
||||
|
||||
@@ -19,6 +19,8 @@ func TestFixtures(t *testing.T) {
|
||||
}{
|
||||
{name: "a take-in's pipeline started with its two movers", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.3589", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"repository\":\"mesh-sdk\",\"tier\":1},{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":2}],\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"repository\":\"mesh-sdk\",\"tier\":1},{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":2}],\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineStartedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-started\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline started again at a re-pin, with the three revisions its movers are kept at (H-A-496)", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.291", given: "{\"at\":\"2026-11-22T20:00:00Z\",\"branch\":\"main\",\"kept\":[292,295,298],\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"pipeline\":\"main.291\",\"revision\":291,\"source\":114,\"v\":1}", wire: "{\"at\":\"2026-11-22T20:00:00Z\",\"branch\":\"main\",\"kept\":[292,295,298],\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"pipeline\":\"main.291\",\"revision\":291,\"source\":114,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineStartedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-started\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline ended passed: per mover its version, steps and tier, an optional step failed", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.3589", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-ended\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline of two movers ended with its phases and one mover left unfinished", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.3589", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"form\":2,\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"phases\":{\"build\":{\"end\":\"2026-11-21T22:02:00Z\",\"start\":\"2026-11-21T22:00:06Z\"},\"composition\":{\"end\":\"2026-11-21T22:06:00Z\",\"start\":\"2026-11-21T22:04:01Z\"},\"ended\":\"2026-11-21T22:06:00Z\",\"placed\":\"2026-11-21T22:00:05Z\",\"taken\":\"2026-11-21T22:00:00Z\",\"unit\":{\"end\":\"2026-11-21T22:04:00Z\",\"start\":\"2026-11-21T22:02:01Z\"}},\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"unfinished\":[\"mesh-delivery-view\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"form\":2,\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"phases\":{\"build\":{\"end\":\"2026-11-21T22:02:00Z\",\"start\":\"2026-11-21T22:00:06Z\"},\"composition\":{\"end\":\"2026-11-21T22:06:00Z\",\"start\":\"2026-11-21T22:04:01Z\"},\"ended\":\"2026-11-21T22:06:00Z\",\"placed\":\"2026-11-21T22:00:05Z\",\"taken\":\"2026-11-21T22:00:00Z\",\"unit\":{\"end\":\"2026-11-21T22:04:00Z\",\"start\":\"2026-11-21T22:02:01Z\"}},\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"unfinished\":[\"mesh-delivery-view\"],\"v\":1}",
|
||||
|
||||
@@ -29,6 +29,29 @@
|
||||
"at": "2026-11-10T10:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a pipeline started again at a re-pin, with the three revisions its movers are kept at (H-A-496)",
|
||||
"of": "event:pipeline-started",
|
||||
"subject": "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.291",
|
||||
"given": {
|
||||
"pipeline": "main.291",
|
||||
"branch": "main",
|
||||
"revision": 291,
|
||||
"source": 114,
|
||||
"v": 1,
|
||||
"movers": [
|
||||
{
|
||||
"module": "nats",
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"fingerprint": "fp-0123",
|
||||
"repository": "mesh-networking",
|
||||
"tier": 1
|
||||
}
|
||||
],
|
||||
"kept": [292, 295, 298],
|
||||
"at": "2026-11-22T20:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a pipeline ended passed: per mover its version, steps and tier, an optional step failed",
|
||||
"of": "event:pipeline-ended",
|
||||
|
||||
@@ -1675,9 +1675,16 @@
|
||||
"published": {
|
||||
"$ref": "#/defs/Published",
|
||||
"description": "for a pipeline a package's publish started, that publish (A-202)"
|
||||
},
|
||||
"kept": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"description": "every catalog revision the pipeline's movers are kept at, said again in full at each re-pin, so the indexer pins each while the pipeline runs (design 47 H2 H-A-496 and its addendum)"
|
||||
}
|
||||
},
|
||||
"description": "a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52)",
|
||||
"description": "a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52), and again at a re-pin with its kept revisions (H2 H-A-496)",
|
||||
"required": [
|
||||
"v",
|
||||
"pipeline",
|
||||
@@ -2367,5 +2374,5 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"form": 8
|
||||
"form": 9
|
||||
}
|
||||
|
||||
@@ -446,7 +446,7 @@ export interface PipelineRow {
|
||||
"steps"?: Record<string, number>;
|
||||
}
|
||||
|
||||
/** a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52) */
|
||||
/** a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52), and again at a re-pin with its kept revisions (H2 H-A-496) */
|
||||
export interface PipelineStarted {
|
||||
"at"?: string;
|
||||
/** the watched branch, or the branch of a run on request */
|
||||
@@ -455,6 +455,8 @@ export interface PipelineStarted {
|
||||
"call"?: string;
|
||||
/** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */
|
||||
"form"?: number;
|
||||
/** every catalog revision the pipeline's movers are kept at, said again in full at each re-pin, so the indexer pins each while the pipeline runs (design 47 H2 H-A-496 and its addendum) */
|
||||
"kept"?: number[];
|
||||
/** the pipeline's movers */
|
||||
"movers": Mover[];
|
||||
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
|
||||
|
||||
@@ -664,7 +664,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
sources.refresh()
|
||||
})
|
||||
// The held take-ins are taken in again from the state held at the start, before a branch's consumer gives a later one
|
||||
// (design 47 H, H-A-114, H-A-118).
|
||||
// (design 47 H, H-A-114, H-A-118); a chain's key a stop left is deleted first (H2 H-A-497 addendum).
|
||||
ix.EndChains(ctx)
|
||||
sources.takeAgain()
|
||||
sources.refresh()
|
||||
if stopped != nil {
|
||||
|
||||
@@ -265,24 +265,34 @@ type actor struct {
|
||||
s *sourceMoved
|
||||
}
|
||||
|
||||
// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending).
|
||||
// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending), beside the asks' loop, and answers
|
||||
// at once (design 47 H2 H-A-497 and its addendum): a chain that runs already answers its state from the state takeins; held
|
||||
// sources start the chain at the first, written there before it runs, answered with how many wait.
|
||||
func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn {
|
||||
r := index.TakenIn{Module: branch}
|
||||
if hs, err := a.HeldOf(ctx, branch); err != nil || len(hs) == 0 {
|
||||
if c, running, err := a.ChainOf(ctx, branch); err != nil {
|
||||
r.Answer = "the state takeins is not read: " + err.Error()
|
||||
return r
|
||||
} else if running {
|
||||
r.Held = true
|
||||
r.Answer = fmt.Sprintf("a chain runs: source %d taken in since %s, %d wait", c.Source, c.Since.Format(time.RFC3339), c.Waiting)
|
||||
return r
|
||||
}
|
||||
hs, err := a.HeldOf(ctx, branch)
|
||||
if err != nil || len(hs) == 0 {
|
||||
r.Answer = "the branch " + branch + " holds no source-moved"
|
||||
if err != nil {
|
||||
r.Answer = "the state held is not read: " + err.Error()
|
||||
}
|
||||
return r
|
||||
}
|
||||
a.s.takeAgain()
|
||||
hs, err := a.HeldOf(ctx, branch)
|
||||
still := err != nil || len(hs) > 0
|
||||
r.Stored, r.Held = !still, still
|
||||
r.Answer = "taken in"
|
||||
if still {
|
||||
r.Answer = "still held: indexer_pending says why"
|
||||
}
|
||||
c := a.StartChain(ctx, hs)
|
||||
go func() {
|
||||
a.s.takeAgain()
|
||||
a.EndChain(a.s.ctx, branch)
|
||||
}()
|
||||
r.Held = true
|
||||
r.Answer = fmt.Sprintf("chain started at source %d, %d wait", c.Source, c.Waiting)
|
||||
return r
|
||||
}
|
||||
|
||||
|
||||
@@ -97,6 +97,98 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// chainStore is a takeins state in memory, safe beside the chain's own goroutine, which says on ended when a chain key goes.
|
||||
type chainStore struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
ended chan string
|
||||
}
|
||||
|
||||
func (c *chainStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
v, ok := c.vals[k]
|
||||
return v, 1, ok, nil
|
||||
}
|
||||
func (c *chainStore) Keys(context.Context) ([]string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var out []string
|
||||
for k := range c.vals {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (c *chainStore) Create(_ context.Context, k string, v []byte) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.vals[k] = v
|
||||
return nil
|
||||
}
|
||||
func (c *chainStore) Update(ctx context.Context, k string, v []byte, _ uint64) error {
|
||||
return c.Create(ctx, k, v)
|
||||
}
|
||||
func (c *chainStore) Delete(_ context.Context, k string, _ uint64) error {
|
||||
c.mu.Lock()
|
||||
_, had := c.vals[k]
|
||||
delete(c.vals, k)
|
||||
c.mu.Unlock()
|
||||
if had && strings.HasPrefix(k, index.ChainPrefix) {
|
||||
c.ended <- k
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-497 and its addendum: a hand ask of a branch with held sources starts the chain beside the asks' loop
|
||||
// and answers at once; an ask that meets the running chain answers its state, which the state takeins holds and
|
||||
// indexer_pending shows; the chain's key goes at its end.
|
||||
func TestAHandAskOfAHeldBranchAnswersAtOnce(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
held := &memStore{vals: map[string][]byte{}}
|
||||
takeIns := &chainStore{vals: map[string][]byte{}, ended: make(chan string, 4)}
|
||||
ix := &index.Indexer{Held: held, TakeIns: takeIns, Logf: func(string, ...any) {},
|
||||
Now: func() time.Time { return time.Date(2026, 11, 22, 21, 7, 0, 0, time.UTC) }}
|
||||
handlers := map[string]func(jetstream.Msg){}
|
||||
s := &sourceMoved{ctx: ctx, ix: ix,
|
||||
branches: func(context.Context) ([]string, error) { return []string{"main"}, nil },
|
||||
bind: func(_ context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
|
||||
handlers[branch] = each
|
||||
return func() {}, nil
|
||||
},
|
||||
logf: func(string, ...any) {}}
|
||||
s.refresh()
|
||||
for _, n := range []int{114, 115, 116} {
|
||||
m := &fakeMsg{subject: fmt.Sprintf("mesh.seat.mesh-delivery.event.source-moved.main.%d", n),
|
||||
data: []byte(fmt.Sprintf(`{"v":1,"branch":"main","source":%d,"call":"%d123456789abcde","heads":[]}`, n, n))}
|
||||
handlers["main"](msgOf{f: m})
|
||||
}
|
||||
a := actor{ix, s}
|
||||
s.again.Lock() // the chain waits for the lock, so it is still running at the second ask
|
||||
if r := a.HeldAgain(ctx, "main"); r.Answer != "chain started at source 114, 2 wait" {
|
||||
s.again.Unlock()
|
||||
t.Fatalf("the first ask answered %+v, want the chain started at once", r)
|
||||
}
|
||||
r := a.HeldAgain(ctx, "main")
|
||||
var pending any
|
||||
var err error
|
||||
for _, tool := range index.HeldTools(held, takeIns, nil, ix.Now, nil) {
|
||||
if tool.Name == index.PendingTool {
|
||||
pending, err = tool.Run(map[string]any{})
|
||||
}
|
||||
}
|
||||
s.again.Unlock()
|
||||
chains := pending.(index.PendingAnswer).Chains
|
||||
if r.Answer != "a chain runs: source 114 taken in since 2026-11-22T21:07:00Z, 2 wait" || !r.Held {
|
||||
t.Errorf("the ask meeting the chain answered %+v, want its state at once", r)
|
||||
}
|
||||
if err != nil || len(chains) != 1 || chains[0].Branch != "main" || chains[0].Source != 114 || chains[0].Waiting != 2 {
|
||||
t.Errorf("indexer_pending shows chains %+v (%v), want main's at 114 with 2 waiting", chains, err)
|
||||
}
|
||||
if k := <-takeIns.ended; k != index.ChainKey("main") {
|
||||
t.Errorf("the key %s went, want main's chain", k)
|
||||
}
|
||||
}
|
||||
|
||||
// revisionObjects is the catalog revisions' object bucket in memory.
|
||||
type revisionObjects map[string][]byte
|
||||
|
||||
|
||||
@@ -33,8 +33,8 @@ type takerIn interface {
|
||||
// written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
|
||||
// answered already (this indexer's own write) or deleted is left.
|
||||
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) {
|
||||
return // a running take-in's waiting-on is the indexer's own record, no ask (H-A-409)
|
||||
if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) || strings.HasPrefix(e.Key(), index.ChainPrefix) {
|
||||
return // a running take-in's waiting-on and a running chain are the indexer's own records, no ask (H-A-409, H-A-497)
|
||||
}
|
||||
a, err := index.DecodeTakeIn(e.Value())
|
||||
if err != nil {
|
||||
|
||||
@@ -51,6 +51,9 @@ type PendingAnswer struct {
|
||||
Held []HeldTakeIn `json:"held"`
|
||||
// Running are the take-ins that run, each with what it waits on (design 47 H2 §803, H-A-409).
|
||||
Running []RunningTakeIn `json:"running,omitempty"`
|
||||
// Chains are the running chains of held take-ins, each branch's source being taken in, since when and how many wait
|
||||
// (H-A-497 addendum).
|
||||
Chains []HeldChain `json:"chains,omitempty"`
|
||||
}
|
||||
|
||||
// ErrHeldUnwritten says a source-moved that could not be taken in is not written into the state held either: its message
|
||||
@@ -167,26 +170,153 @@ func (ix *Indexer) unhold(ctx context.Context, branch string, source int64) {
|
||||
}
|
||||
}
|
||||
|
||||
// TakeHeldIn takes every branch's held source revisions in, each branch in its order, on the indexer's own room and at its
|
||||
// start (H-A-114, H-A-118): a branch stops at the first that still cannot be taken in, which stays held with those behind it.
|
||||
// TakeHeldIn takes every branch's held source revisions in, each branch in its order, on the indexer's own room, at its
|
||||
// start and by hand (H-A-114, H-A-118): a branch stops at the first that still cannot be taken in, which stays held with those
|
||||
// behind it. Once a held one is taken in, the next held of its branch is taken in at once, each such start said in the journal
|
||||
// (design 47 H2 H-A-497).
|
||||
func (ix *Indexer) TakeHeldIn(ctx context.Context) {
|
||||
all, err := ReadHeld(ctx, ix.Held)
|
||||
if err != nil {
|
||||
ix.logf("take-in: the state %s is not read, so no held take-in is taken in again now: %v", HeldState, err)
|
||||
return
|
||||
}
|
||||
stopped := map[string]bool{}
|
||||
stopped, landed := map[string]bool{}, map[string]int64{}
|
||||
behind := map[string]int{}
|
||||
for _, h := range all {
|
||||
behind[h.Branch]++
|
||||
}
|
||||
for i, h := range all {
|
||||
behind[h.Branch]--
|
||||
if stopped[h.Branch] {
|
||||
continue
|
||||
}
|
||||
if _, err := ix.TakeInSource(ctx, h.Event()); err != nil {
|
||||
if prev, ok := landed[h.Branch]; ok {
|
||||
ix.logf("take-in: held source %d of %s taken in next, after %d (H-A-497)", h.Source, h.Branch, prev)
|
||||
}
|
||||
ix.writeChain(ctx, HeldChain{Branch: h.Branch, Source: h.Source, Since: ix.now().UTC(), Waiting: behind[h.Branch]})
|
||||
if _, err := ix.TakeInSource(ctx, h.Event()); err == nil {
|
||||
landed[h.Branch] = h.Source
|
||||
} else {
|
||||
ix.logf("take-in: source revision %d of %s is still held: %v", h.Source, h.Branch, err)
|
||||
stopped[h.Branch] = true
|
||||
}
|
||||
// The chain of a branch ends at its first held again or its last held (H-A-497 addendum).
|
||||
if stopped[h.Branch] || !slices.ContainsFunc(all[i+1:], func(n HeldTakeIn) bool { return n.Branch == h.Branch }) {
|
||||
ix.endChain(ctx, h.Branch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ChainPrefix is the takeins keys of the chains of held take-ins: `chain.<branch>`, the branch escaped as a key's name is,
|
||||
// holding the branch's chain while it runs (design 47 H2 H-A-497 and its addendum): written at each step, deleted at its end.
|
||||
const ChainPrefix = "chain."
|
||||
|
||||
// ChainKey is the takeins key of branch's chain.
|
||||
func ChainKey(branch string) string { return ChainPrefix + keys.EscapeName(branch) }
|
||||
|
||||
// HeldChain is a running chain of a branch's held take-ins (H-A-497 addendum): the source being taken in, since when, and
|
||||
// how many held sources of the branch wait behind it.
|
||||
type HeldChain struct {
|
||||
Branch string `json:"branch"`
|
||||
Source int64 `json:"source"`
|
||||
Since time.Time `json:"since"`
|
||||
Waiting int `json:"waiting"`
|
||||
}
|
||||
|
||||
// writeChain writes branch's chain at its step c into the state takeins; a failed write is said.
|
||||
func (ix *Indexer) writeChain(ctx context.Context, c HeldChain) {
|
||||
if ix.TakeIns == nil {
|
||||
return
|
||||
}
|
||||
data, err := json.Marshal(c)
|
||||
if err == nil {
|
||||
err = write(ctx, ix.TakeIns, ChainKey(c.Branch), func([]byte, bool) ([]byte, bool, error) { return data, false, nil })
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("take-in: the chain of %s at source revision %d is not written into %s: %v", c.Branch, c.Source, TakeInsState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// EndChain deletes branch's chain from the state takeins once the take-in its ask started returned, so a chain whose held
|
||||
// sources another act took first is never left running.
|
||||
func (ix *Indexer) EndChain(ctx context.Context, branch string) { ix.endChain(ctx, branch) }
|
||||
|
||||
// EndChains deletes every chain the state takeins holds, at the indexer's start (H-A-497 addendum): no chain runs before the
|
||||
// start's take-in, so a key left by a stop between a chain's last take-in and its end would answer "a chain runs" for good.
|
||||
func (ix *Indexer) EndChains(ctx context.Context) {
|
||||
chains, err := ReadChains(ctx, ix.TakeIns)
|
||||
if err != nil {
|
||||
ix.logf("take-in: the chains in %s are not read, so none left by a stop is deleted: %v", TakeInsState, err)
|
||||
return
|
||||
}
|
||||
for _, c := range chains {
|
||||
ix.logf("take-in: the chain of %s at source revision %d, left by a stop, is deleted at the start", c.Branch, c.Source)
|
||||
ix.endChain(ctx, c.Branch)
|
||||
}
|
||||
}
|
||||
|
||||
// endChain deletes branch's chain from the state takeins.
|
||||
func (ix *Indexer) endChain(ctx context.Context, branch string) {
|
||||
if ix.TakeIns == nil {
|
||||
return
|
||||
}
|
||||
if err := write(ctx, ix.TakeIns, ChainKey(branch), func([]byte, bool) ([]byte, bool, error) { return nil, true, nil }); err != nil {
|
||||
ix.logf("take-in: the chain of %s is not deleted from %s: %v", branch, TakeInsState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// StartChain writes branch's chain at its first held source, before the chain is taken in beside the asks (H-A-497
|
||||
// addendum), so an ask after it finds it running; it answers that chain.
|
||||
func (ix *Indexer) StartChain(ctx context.Context, held []HeldTakeIn) HeldChain {
|
||||
c := HeldChain{Branch: held[0].Branch, Source: held[0].Source, Since: ix.now().UTC(), Waiting: len(held) - 1}
|
||||
ix.writeChain(ctx, c)
|
||||
return c
|
||||
}
|
||||
|
||||
// ChainOf is branch's running chain in the state takeins, and whether one runs.
|
||||
func (ix *Indexer) ChainOf(ctx context.Context, branch string) (HeldChain, bool, error) {
|
||||
if ix.TakeIns == nil {
|
||||
return HeldChain{}, false, nil
|
||||
}
|
||||
v, _, found, err := ix.TakeIns.Get(ctx, ChainKey(branch))
|
||||
if err != nil || !found {
|
||||
return HeldChain{}, false, err
|
||||
}
|
||||
var c HeldChain
|
||||
if err := json.Unmarshal(v, &c); err != nil {
|
||||
return HeldChain{}, false, fmt.Errorf("%s.%s: %w", TakeInsState, ChainKey(branch), err)
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
|
||||
// ReadChains are the running chains the state takeins holds (H-A-497 addendum), each branch's once.
|
||||
func ReadChains(ctx context.Context, st Store) ([]HeldChain, error) {
|
||||
if st == nil {
|
||||
return nil, nil
|
||||
}
|
||||
ks, err := st.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(ks)
|
||||
var out []HeldChain
|
||||
for _, k := range ks {
|
||||
if !strings.HasPrefix(k, ChainPrefix) {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := st.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var c HeldChain
|
||||
if !found || json.Unmarshal(v, &c) != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SkipTakeIn ends a held source revision unwritten by hand (indexer_take_in_skip, §744): it says take-in.taken with no entries,
|
||||
// skipped, by and why, and deletes the branch's held take-in; the caller acknowledges the source-moved.
|
||||
func (ix *Indexer) SkipTakeIn(ctx context.Context, e meshdelivery.SourceMovedEvent, by, why string) error {
|
||||
@@ -236,7 +366,7 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs
|
||||
{Name: PendingTool,
|
||||
Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " +
|
||||
"could not take in (the catalog store full, a read of the forge unanswered), why and since when, in order, each kept here once " +
|
||||
"acknowledged and taken in again on the indexer's room and at its start; and each running take-in that waits over a minute on another part, its waiting-on (part, verb, since). indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.",
|
||||
"acknowledged and taken in again on the indexer's room and at its start; each running take-in that waits over a minute on another part, its waiting-on (part, verb, since); and each running chain of held take-ins, the source being taken in, since when and how many wait. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
h, err := ReadHeld(context.Background(), held)
|
||||
@@ -244,11 +374,16 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs
|
||||
return PendingAnswer{Held: h}, err
|
||||
}
|
||||
r, err := ReadRunning(context.Background(), takeIns)
|
||||
return PendingAnswer{Held: h, Running: r}, err
|
||||
if err != nil {
|
||||
return PendingAnswer{Held: h, Running: r}, err
|
||||
}
|
||||
c, err := ReadChains(context.Background(), takeIns)
|
||||
return PendingAnswer{Held: h, Running: r, Chains: c}, err
|
||||
}},
|
||||
{Name: TakeInPendingTool,
|
||||
Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " +
|
||||
"answers whether it was taken in. indexer_pending names the held branches.",
|
||||
"the held sources are taken in in order beside the asks, and it answers at once that the chain started at a source and " +
|
||||
"how many wait, or, while a chain runs, its state (design 47 H2 H-A-497). indexer_pending names the held branches and the running chains.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"branch"}, "properties": map[string]any{
|
||||
"branch": map[string]any{"type": "string", "description": "the watched branch, as indexer_pending names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken again now"}}},
|
||||
|
||||
@@ -73,8 +73,8 @@ func pinObject(key string) (string, bool) {
|
||||
}
|
||||
|
||||
// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a
|
||||
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it and every
|
||||
// other revision the record pins (H-A-147). A body
|
||||
// pipeline-started pins the revision it names, and each revision its movers are kept at (design 47 H2 H-A-496), under the
|
||||
// record `pipeline.<pipeline>`, its pipeline-ended frees it and every other revision the record pins (H-A-147). A body
|
||||
// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it
|
||||
// again; one applied twice changes nothing.
|
||||
func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error {
|
||||
@@ -83,7 +83,15 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
}
|
||||
switch {
|
||||
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
||||
if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)); err != nil {
|
||||
record := PipelineRecord(started.Pipeline)
|
||||
if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), record); err != nil {
|
||||
return err
|
||||
}
|
||||
// Every revision the pipeline's movers are kept at is pinned under its record too, so no judge removes one it still
|
||||
// reads (design 47 H2 H-A-496); the latest started's kept set is the pipeline's kept pins, so a kept revision the
|
||||
// record pins that this started no longer names is freed (H-A-496 addendum). A kept revision is always newer than the
|
||||
// pipeline's own (a re-pin of its own source, H-A-283), so its own revision and its validate base stay pinned.
|
||||
if err := ix.pinKept(ctx, started.Branch, started.Revision, started.Kept, record); err != nil {
|
||||
return err
|
||||
}
|
||||
// The reservations of the pipeline's revision settle at its start, if they gather (design 47 H2 §805, H-A-424 (1)).
|
||||
@@ -229,6 +237,41 @@ func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// pinKept pins each of kept, the revisions a pipeline of branch at revision own keeps movers at, under record, and frees
|
||||
// every revision newer than own the record pins that kept does not name (design 47 H2 H-A-496 and its addendum).
|
||||
func (ix *Indexer) pinKept(ctx context.Context, branch string, own int64, kept []int64, record string) error {
|
||||
for _, r := range kept {
|
||||
if r <= 0 || r == own {
|
||||
continue
|
||||
}
|
||||
if err := ix.pin(ctx, PinKey(branch, r), record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
b, r, ok := ParsePinKey(k)
|
||||
if !ok || b != branch || r <= own || slices.Contains(kept, r) {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pv PinValue
|
||||
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
|
||||
continue
|
||||
}
|
||||
if err := ix.unpin(ctx, k, record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PipelineRecord is the record a pipeline pins its revision under: `pipeline.<branch>.<source>` (design 47 H §743, A-449a).
|
||||
func PipelineRecord(pipeline string) string { return "pipeline." + pipeline }
|
||||
|
||||
|
||||
@@ -13,6 +13,47 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 H2 H-A-496 and its addendum: a pipeline-started pins the pipeline's own revision and each revision its movers
|
||||
// are kept at under the pipeline's record; a later started (said again at a re-pin) is the pipeline's kept set, freeing a
|
||||
// kept revision it no longer names but never the pipeline's own; pipeline-ended frees them all.
|
||||
func TestAPipelinesKeptRevisionsArePinnedUntilItEnds(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pending := uint64(1) // the keeps judge nothing while the consumer is behind
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
|
||||
Now: func() time.Time { return time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC) }, PinsPending: func(context.Context) (uint64, error) { return pending, nil }}
|
||||
kept := func() []string {
|
||||
n, _ := ix.Revisions.Names(ctx)
|
||||
return slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok })
|
||||
}
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||
for source := int64(1); source <= 6; source++ {
|
||||
e := reader
|
||||
e.Commit = fmt.Sprintf("c%d", source)
|
||||
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pending = 0
|
||||
heard := func(started *runner.PipelineStarted, ended *runner.PipelineEnded, want ...string) {
|
||||
t.Helper()
|
||||
if err := ix.HeardPins(ctx, started, ended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
if got := kept(); !slices.Equal(got, want) {
|
||||
t.Errorf("kept %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{2, 3, 4}}, nil,
|
||||
"main.1", "main.2", "main.3", "main.4", "main.6")
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, "main.3"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.1"}) {
|
||||
t.Errorf("the pin of main.3 holds %s, want the pipeline's record", v)
|
||||
}
|
||||
heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{3, 4}}, nil,
|
||||
"main.1", "main.3", "main.4", "main.6")
|
||||
heard(nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.1", Revision: 1, Outcome: runner.PipelineEndedOutcomeFailed}, "main.6")
|
||||
}
|
||||
|
||||
// Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its
|
||||
// pipeline-ended, counts a record once, and removes a revision only while its consumers are caught up, judged against the pins
|
||||
// its own state holds, never a count of pins stated: one two pipelines pin stays until both free it, one neither pinned nor
|
||||
|
||||
@@ -101,9 +101,10 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said)
|
||||
}
|
||||
|
||||
// Design 47 H2 §805, H-A-418, H-A-420, H-A-424, H-A-436: a pipeline reserving twenty packages of mesh-sdk in main's
|
||||
// revision 2 keeps one re-pin revision of main when its gathering settles, at its first step-ended; its release then
|
||||
// keeps none on main, and a pipeline that did not pass withdraws its reservations back to the newest released version.
|
||||
// Design 47 H2 H-A-495 (ending H-A-418 (2), H-A-424 (4)), H-A-436: a pin names only a released version. A pipeline
|
||||
// reserving twenty packages of mesh-sdk in main's revision 2 moves no pin when its gathering settles; the release of
|
||||
// the reserved version re-pins g in one revision; a pipeline that did not pass withdraws its reservations and moves
|
||||
// nothing; and a pin held on a version never released is moved to the newest released at the next re-pin, said.
|
||||
ix.Files = func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) {
|
||||
out := map[string]map[string]string{}
|
||||
for _, d := range dirs {
|
||||
@@ -111,6 +112,8 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
var lines []string
|
||||
ix.Logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) }
|
||||
heard := newMem()
|
||||
ix.Heard = heard
|
||||
revision := BranchObject("main", 2)
|
||||
@@ -121,6 +124,7 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
}
|
||||
return rev.Revision
|
||||
}
|
||||
const sdkGo = "git.novox.be/novox/mesh-sdk/go"
|
||||
for i := range 20 {
|
||||
e := meshbuilds.VersionReservedEvent{Module: "mesh-sdk", Artifact: fmt.Sprintf("mesh-sdk:p%d", i), Path: "mesh-sdk/go", Version: "0.1.28", Revision: revision}
|
||||
if err := ix.HearVersionReserved(ctx, e); err != nil {
|
||||
@@ -130,44 +134,57 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
if _, _, open, _ := heard.Get(ctx, SettlingKey(revision)); !open || newest() != 2 {
|
||||
t.Fatalf("twenty reservations: gathering open %v, main's newest %d, want open and no revision kept before it settles", open, newest())
|
||||
}
|
||||
if err := ix.Settle(ctx, revision); err != nil || newest() != 3 {
|
||||
t.Fatalf("the gathering settled (%v) keeping main's newest %d, want one re-pin revision, 3", err, newest())
|
||||
if err := ix.Settle(ctx, revision); err != nil || newest() != 2 {
|
||||
t.Fatalf("the gathering settled (%v) keeping main's newest %d, want still 2: a reservation moves no pin", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" {
|
||||
t.Errorf("g after the reservations is pinned %v, want v0.1.28", now.Packages)
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.27" {
|
||||
t.Errorf("g after the reservations is pinned %v, want the released v0.1.27", now.Packages)
|
||||
}
|
||||
if err := ix.Settle(ctx, revision); err != nil || newest() != 3 {
|
||||
t.Errorf("a second step-ended settled (%v) keeping main's newest %d, want still 3", err, newest())
|
||||
}
|
||||
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.28", Commit: strings.Repeat("1", 40), Published: true}); err != nil {
|
||||
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: sdkGo, Version: "0.1.28", Commit: strings.Repeat("1", 40), Published: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ix.Repin(ctx, "mesh-sdk", "0.1.28"); err != nil || newest() != 3 {
|
||||
t.Errorf("the release of the reserved 0.1.28 re-pinned (%v) keeping main's newest %d, want no revision more", err, newest())
|
||||
t.Errorf("the release of the reserved 0.1.28 re-pinned (%v) keeping main's newest %d, want one revision more, 3", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.28" {
|
||||
t.Errorf("g after the release is pinned %v, want v0.1.28", now.Packages)
|
||||
}
|
||||
next := BranchObject("main", 3)
|
||||
if err := ix.HearVersionReserved(ctx, meshbuilds.VersionReservedEvent{Module: "mesh-sdk", Artifact: "mesh-sdk:p0", Path: "mesh-sdk/go", Version: "0.1.29", Revision: next}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Settle(ctx, next); err != nil || newest() != 4 {
|
||||
t.Fatalf("the reservation of 0.1.29 settled (%v) keeping main's newest %d, want 4", err, newest())
|
||||
if err := ix.Settle(ctx, next); err != nil || newest() != 3 {
|
||||
t.Fatalf("the reservation of 0.1.29 settled (%v) keeping main's newest %d, want still 3", err, newest())
|
||||
}
|
||||
// H-A-479 second addendum: a pipeline at the same revision whose movers are others withdraws none of mesh-sdk's.
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"app"}); err != nil || newest() != 4 {
|
||||
t.Fatalf("another pipeline's withdrawal (%v) kept main's newest %d, want 4: nothing of mesh-sdk withdrawn", err, newest())
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"app"}); err != nil || newest() != 3 {
|
||||
t.Fatalf("another pipeline's withdrawal (%v) kept main's newest %d, want 3: nothing of mesh-sdk withdrawn", err, newest())
|
||||
}
|
||||
if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); !kept {
|
||||
t.Fatalf("another pipeline's withdrawal took mesh-sdk's reservation")
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"mesh-sdk"}); err != nil || newest() != 5 {
|
||||
t.Fatalf("the withdrawal (%v) kept main's newest %d, want 5", err, newest())
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"mesh-sdk"}); err != nil || newest() != 3 {
|
||||
t.Fatalf("the withdrawal (%v) kept main's newest %d, want 3: the withdrawn reservation pinned nothing", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" {
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.28" {
|
||||
t.Errorf("g after the withdrawal is pinned %v, want the newest released v0.1.28", now.Packages)
|
||||
}
|
||||
if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); kept {
|
||||
t.Errorf("the withdrawn reservation is still kept")
|
||||
}
|
||||
stale, _ := moduleOf(t, mods, "g")
|
||||
stale.Packages = map[string]string{sdkGo: "v0.1.29"} // pinned before H-A-495 to the withdrawn reservation
|
||||
mods.put("g", mustEncode(t, stale))
|
||||
lines = nil
|
||||
if _, err := ix.Repin(ctx, "mesh-sdk", "0.1.28"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages[sdkGo] != "v0.1.28" {
|
||||
t.Errorf("g pinned to the never released v0.1.29 is pinned %v after a re-pin, want v0.1.28", now.Packages)
|
||||
}
|
||||
if said := "g: pinned to " + sdkGo + " v0.1.29, never released; moved to v0.1.28"; !slices.Contains(lines, said) {
|
||||
t.Errorf("the re-pin said %q, want the line %q", lines, said)
|
||||
}
|
||||
// H-A-479 second addendum: two pipelines' movers reserve at one revision while its gathering is open; withdrawing one
|
||||
// pipeline's mover keeps the gathering and the other's reservation, withdrawing the last removes the gathering too.
|
||||
six := BranchObject("main", 6)
|
||||
|
||||
@@ -17,13 +17,13 @@ import (
|
||||
|
||||
// A release re-pins once, at its reservation (design 47 H2 §805; H-A-418, H-A-420, H-A-424). The builder says
|
||||
// version-reserved when it reserves a package's number, naming the revision `<branch>.<n>` its pipeline runs. The indexer
|
||||
// writes the series' numbered version with the contents read at the commit of the module's entry in that revision, so
|
||||
// acceptedSeries and releasedPins pin it, and keeps the reservation in its state heard under
|
||||
// writes the series' numbered version with the contents read at the commit of the module's entry in that revision, and
|
||||
// keeps the reservation in its state heard under
|
||||
// `reserved.<revision>.<module>`, each key living reserved-keep. It gathers a pipeline's reservations per revision and
|
||||
// writes one re-pin revision of that revision's branch when the gathering settles: at the first pipeline-started or
|
||||
// step-ended of that pipeline heard after them, or at the expiry of the key settling.<revision>, written reserved-settle
|
||||
// after the first. The release's package-published then
|
||||
// moves no fingerprint. A pipeline-ended not passed whose revision holds reservations withdraws them: the series'
|
||||
// after the first. A pin names only a released version, never a reservation (design 47 H2 H-A-495), so that re-pin
|
||||
// moves no pin, and the release's re-pin (H-A-165) moves the importers. A pipeline-ended not passed whose revision holds reservations withdraws them: the series'
|
||||
// numbered goes back to its newest released version and the importers are re-pinned to it.
|
||||
|
||||
// ReservedPrefix begins every key of the state heard holding a revision's reservations of one module.
|
||||
|
||||
@@ -198,14 +198,10 @@ func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repos
|
||||
return out, pins
|
||||
}
|
||||
|
||||
// pinnedVersion is the version of a series its users are pinned to: its numbered one when a reservation wrote it with its
|
||||
// contents and it is newer than the newest released (design 47 H2 §805, H-A-418 (2), H-A-424 (4)), else the newest; nil
|
||||
// for none.
|
||||
// pinnedVersion is the version of a series its users are pinned to: its newest released one, never a reservation's
|
||||
// numbered one, which a pipeline that does not pass withdraws unpublished (design 47 H2 H-A-495, which ends H-A-418 (2)
|
||||
// and H-A-424 (4)); nil for none.
|
||||
func pinnedVersion(s SeriesPackage) *SeriesVersion {
|
||||
if n := s.Numbered; n != nil && n.Contents != nil && (s.Newest == nil || CompareVersions("v"+s.Newest.Version, "v"+n.Version) < 0) {
|
||||
p := publishedOfNumbered(*n.Contents)
|
||||
return &SeriesVersion{Version: n.Version, Commit: n.Commit, Digest: n.Digest, Fingerprint: n.Fingerprint, Contents: contentsOfPublished(p)}
|
||||
}
|
||||
return s.Newest
|
||||
}
|
||||
|
||||
|
||||
@@ -371,11 +371,35 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
|
||||
len(held[0].Heads) != 1 || held[0].Heads[0] != moved.Heads[0] || !reflect.DeepEqual(held[0].Event(), moved) {
|
||||
t.Fatalf("held = %+v, want 3 then 4, each with its event whole", held)
|
||||
}
|
||||
// H-A-497: behind them a source whose head the forge does not answer, and one more behind it.
|
||||
for _, n := range []struct {
|
||||
source int64
|
||||
commit string
|
||||
}{{5, "c-unread"}, {6, f.Merge}} {
|
||||
e := moved
|
||||
e.Source, e.Call, e.Heads = n.source, fmt.Sprintf("%d123456789abcdef", n.source), []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: n.commit}}
|
||||
if behind, err := ix.HoldBehind(ctx, e); err != nil || !behind {
|
||||
t.Fatalf("source %d held behind: %v (%v)", n.source, behind, err)
|
||||
}
|
||||
}
|
||||
var lines []string
|
||||
ix.Logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) }
|
||||
ix.TakeIns = newMem()
|
||||
// H-A-497 addendum: while the chain runs, its key names the source being taken in and how many wait behind it.
|
||||
var steps []string
|
||||
read := ix.Read
|
||||
ix.Read = func(ctx context.Context, repository, sha, dir string) (RepoRead, error) {
|
||||
if c, ok, _ := ix.ChainOf(ctx, "main"); ok {
|
||||
steps = append(steps, fmt.Sprintf("%d/%d", c.Source, c.Waiting))
|
||||
}
|
||||
return read(ctx, repository, sha, dir)
|
||||
}
|
||||
objects.capacity = 1 << 22
|
||||
if ok, err := ix.RoomAgain(ctx); err != nil || !ok || rooms != 1 {
|
||||
t.Fatalf("room said %v %d times (%v), want once the store has room", ok, rooms, err)
|
||||
}
|
||||
// H-A-114: on room (and at start) the held ones are taken in again from the state held alone, in their order.
|
||||
// H-A-114, H-A-497: on room (and at start, and by hand) the held ones are taken in again from the state held alone, in
|
||||
// their order, each next one at once and said in the journal, until one is held again: those behind it stay held.
|
||||
ix.TakeHeldIn(ctx)
|
||||
var taken []int64
|
||||
for _, e := range *said {
|
||||
@@ -386,8 +410,38 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
|
||||
if !reflect.DeepEqual(taken, []int64{3, 4}) {
|
||||
t.Errorf("taken again on room: %v of %+v, want 3 then 4", taken, *said)
|
||||
}
|
||||
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 {
|
||||
t.Errorf("held = %+v after the take-in, want none", held)
|
||||
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 2 || held[0].Source != 5 || held[1].Source != 6 {
|
||||
t.Errorf("held = %+v after the take-in, want 5 held again and 6 behind it", held)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"take-in: held source 4 of main taken in next, after 3 (H-A-497)",
|
||||
"take-in: held source 5 of main taken in next, after 4 (H-A-497)",
|
||||
} {
|
||||
if !slices.Contains(lines, want) {
|
||||
t.Errorf("the journal holds no line %q: %q", want, lines)
|
||||
}
|
||||
}
|
||||
for _, l := range lines {
|
||||
if strings.Contains(l, "held source 6 of main taken in") {
|
||||
t.Errorf("source 6 was taken in behind a held one: %q", l)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(steps, "4/2") || !slices.Contains(steps, "5/1") {
|
||||
t.Errorf("the chain's key read at each take-in %v, want main at 4 with 2 waiting and at 5 with 1", steps)
|
||||
}
|
||||
// H-A-497 addendum: the chain's key in the state takeins goes at its end, here its first held again, and one a stop
|
||||
// left is deleted at the start.
|
||||
if chains, err := ReadChains(ctx, ix.TakeIns); err != nil || len(chains) != 0 {
|
||||
t.Errorf("chains %+v (%v) after the chain stopped, want none", chains, err)
|
||||
}
|
||||
ix.writeChain(ctx, HeldChain{Branch: "main", Source: 9, Waiting: 1})
|
||||
if ix.EndChains(ctx); func() bool { c, _ := ReadChains(ctx, ix.TakeIns); return len(c) != 0 }() {
|
||||
t.Error("a chain's key left by a stop stays after the start")
|
||||
}
|
||||
ix.Read = read
|
||||
ix.Logf, ix.TakeIns = nil, nil
|
||||
for _, src := range []int64{5, 6} {
|
||||
ix.unhold(ctx, "main", src)
|
||||
}
|
||||
// A skip ends a held source revision unwritten: take-in.taken with no entries, skipped, by and why; held is deleted.
|
||||
ix.hold(ctx, meshdelivery.SourceMovedEvent{Branch: "nox-mesh", Source: 9, Call: "fedcba9876543210"}, "the catalog store is full")
|
||||
|
||||
@@ -236,6 +236,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
fp, none string
|
||||
tcs map[string]meshcatalog.ToolchainUsed // by language, when there is a fingerprint
|
||||
pkgs sdkstep.PackageVersions // the package versions, when there is a fingerprint
|
||||
moved []string // the pins moved off a version never released, said (H-A-495)
|
||||
}
|
||||
done := map[string]result{}
|
||||
visiting := map[string]bool{}
|
||||
@@ -313,6 +314,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
// a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
|
||||
r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository, kept.Packages)
|
||||
r.tcs = toolchainsUsed(toolchains)
|
||||
r.moved = neverReleased(module, k.held.Packages, r.pkgs, series, k.manifest)
|
||||
}
|
||||
}
|
||||
if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
|
||||
@@ -325,6 +327,9 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
}
|
||||
done[module] = r
|
||||
k.cur.Inputs2 = &kept
|
||||
for _, line := range r.moved {
|
||||
ix.logf("%s", line)
|
||||
}
|
||||
return r
|
||||
}
|
||||
finals := map[string]meshcatalog.ModulesByModuleValue{}
|
||||
|
||||
@@ -140,7 +140,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
|
||||
}
|
||||
var want []string
|
||||
for _, a := range entry.Artifacts {
|
||||
want = append(want, a.Ref)
|
||||
if a.Kind != catalogue.ArtifactUpstream { // an upstream copy is released by its pinned digest, never published (H-A-486)
|
||||
want = append(want, a.Ref)
|
||||
}
|
||||
}
|
||||
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
|
||||
var have published
|
||||
@@ -211,7 +213,15 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
|
||||
at := ix.now().UTC()
|
||||
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
|
||||
pins, err := upstreamPins(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if a.Kind == catalogue.ArtifactUpstream {
|
||||
v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: a.Ref, Digest: pins[a.Ref]})
|
||||
continue
|
||||
}
|
||||
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
|
||||
}
|
||||
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
|
||||
@@ -255,6 +265,46 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
return ix.forgetPublished(ctx, pendingKey)
|
||||
}
|
||||
|
||||
// upstreamPins are the pinned digests of the entry's upstream artifacts, by reference, read from its manifest's from
|
||||
// (`<image>@sha256:<digest>`): design 51 copies an upstream image by that digest, so a release names it (H-A-486). An upstream
|
||||
// artifact whose manifest names no pin refuses the release, naming it.
|
||||
func upstreamPins(entry meshcatalog.CatalogEntry) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
if !slices.ContainsFunc(entry.Artifacts, func(a meshcatalog.Artifact) bool { return a.Kind == catalogue.ArtifactUpstream }) {
|
||||
return out, nil
|
||||
}
|
||||
raw, err := json.Marshal(entry.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
From string `json:"from"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s's entry does not read, so its upstream artifacts' pins are not known: %w", entry.Module, err)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactUpstream {
|
||||
continue
|
||||
}
|
||||
if _, d, ok := strings.Cut(a.From, "@"); ok && strings.HasPrefix(d, "sha256:") {
|
||||
out[ArtifactRefOf(entry.Module, a.Name)] = d
|
||||
}
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if _, ok := out[a.Ref]; a.Kind == catalogue.ArtifactUpstream && !ok {
|
||||
return nil, fmt.Errorf("%s is an upstream artifact whose manifest names no pinned digest (from <image>@sha256:<digest>), so %s releases no version", a.Ref, entry.Module)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
|
||||
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
|
||||
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
|
||||
@@ -423,6 +473,29 @@ func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, rep
|
||||
return fingerprint.WithPins(f.fp, pins), pins
|
||||
}
|
||||
|
||||
// neverReleased are the lines said of a module whose held pins (was) name a version above its series' newest released one,
|
||||
// a reservation withdrawn unpublished, and whose new pins move it (design 47 H2 H-A-495): "<module>: pinned to <package>
|
||||
// <version>, never released; moved to <version>". nil for none.
|
||||
func neverReleased(module string, was map[string]string, pins sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, p := range slices.Sorted(maps.Keys(was)) {
|
||||
old, now := strings.TrimPrefix(was[p], "v"), pins[p]
|
||||
if _, ok := versionParts(old); !ok || was[p] == now {
|
||||
continue
|
||||
}
|
||||
for _, s := range series {
|
||||
if s.GoModule != p || s.Newest == nil || s.Series != m.AcceptedSeries(s.Name) {
|
||||
continue
|
||||
}
|
||||
if in, _ := SeriesOf(old); in == s.Series && compareVersions(old, strings.TrimPrefix(s.Newest.Version, "v")) > 0 {
|
||||
out = append(out, fmt.Sprintf("%s: pinned to %s %s, never released; moved to %s", module, p, was[p], cmp.Or(now, "no version")))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moduleOfKey is the module a modules key names; "" for none.
|
||||
func moduleOfKey(key string) string {
|
||||
if key == "" {
|
||||
|
||||
@@ -83,6 +83,36 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
|
||||
if compareVersions(oldest, "0.1.5") < 0 {
|
||||
t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest)
|
||||
}
|
||||
|
||||
// H-A-486: an upstream artifact is copied by its pinned digest and never published, so the version is released once every
|
||||
// other artifact is, the upstream one named with the pin its manifest gives; one whose manifest names no pin releases none.
|
||||
pin := "sha256:" + strings.Repeat("ab", 32)
|
||||
nats := func(from string) map[string]meshcatalog.CatalogEntry {
|
||||
return map[string]meshcatalog.CatalogEntry{"nats": {Module: "nats", Manifest: map[string]any{"build": map[string]any{"artifacts": []any{
|
||||
map[string]any{"name": "server", "kind": "upstream", "from": from},
|
||||
map[string]any{"name": "tools", "kind": "bundle", "from": "."}}}},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "nats:server", Kind: "upstream"}, {Ref: "nats:tools", Kind: "bundle", Language: "go"}}}}
|
||||
}
|
||||
for i, c := range []struct {
|
||||
from, version string
|
||||
released bool
|
||||
}{{"nats:2.12-alpine@" + pin, "0.1.1", true}, {"nats:2.12-alpine", "0.1.2", false}} {
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: int64(4 + i), Kind: "trunk"}, nats(c.from))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "nats", Artifact: "nats:tools", Digest: "sha256:tools", Version: c.version, Source: 9, Branch: "main"})
|
||||
got, _ := ModuleVersions(ctx, ix.Versions, "nats")
|
||||
switch {
|
||||
case c.released && (err != nil || len(got) != 1 || len(got[0].Artifacts) != 2 || got[0].Artifacts[0] != (meshcatalog.VersionArtifact{Artifact: "nats:server", Digest: pin})):
|
||||
t.Fatalf("pinned %q: released %+v (%v); want the server named with its pin", c.from, got, err)
|
||||
case !c.released && (err == nil || !strings.Contains(err.Error(), "no pinned digest") || len(got) != 1):
|
||||
t.Fatalf("unpinned %q: released %+v (%v); want a refusal naming the pin", c.from, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -150,10 +151,17 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi
|
||||
for {
|
||||
raw, err := call("seat:"+catalog.Seat+"."+catalog.VerbRevision, in)
|
||||
if err != nil {
|
||||
// A revision the catalog answers "no such object" for is gone (design 47 H2 H-A-496): a kept mover's wait on it ends.
|
||||
if strings.Contains(err.Error(), "no such object") {
|
||||
return pipeline.Half{}, fmt.Errorf("%w: %v", pipeline.ErrRevisionGone, err)
|
||||
}
|
||||
return pipeline.Half{}, err
|
||||
}
|
||||
answer, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
if strings.Contains(said, "no such object") {
|
||||
return pipeline.Half{}, fmt.Errorf("%w: %s answered: %s", pipeline.ErrRevisionGone, catalog.VerbRevision, said)
|
||||
}
|
||||
return pipeline.Half{}, fmt.Errorf("%s answered: %s", catalog.VerbRevision, said)
|
||||
}
|
||||
var a catalog.RevisionAnswer
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -63,6 +64,19 @@ func TestCatalogRevisionIsKeptOnceRead(t *testing.T) {
|
||||
if h, err := read("main", 0); err != nil || calls != 3 || len(h.Entries) != 2 {
|
||||
t.Fatalf("the newest made calls %d, %+v, %v; want its first page only", calls, h, err)
|
||||
}
|
||||
// A revision the catalog answers "no such object" for is gone, whether refused or answered so (design 47 H2 H-A-496).
|
||||
for _, gone := range []func(string, any) (json.RawMessage, error){
|
||||
func(string, any) (json.RawMessage, error) {
|
||||
return nil, fmt.Errorf("revision main.292: no such object")
|
||||
},
|
||||
func(string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"content":[{"text":"revision main.292: no such object"}],"isError":true}`), nil
|
||||
},
|
||||
} {
|
||||
if _, err := catalogRevision(gone, newRevisionCache(1))("main", 292); !errors.Is(err, pipeline.ErrRevisionGone) {
|
||||
t.Errorf("a revision the catalog has no object of reads %v, want it gone", err)
|
||||
}
|
||||
}
|
||||
c := newRevisionCache(1)
|
||||
c.put("main", pipeline.Half{Revision: 1})
|
||||
c.put("main", pipeline.Half{Revision: 2})
|
||||
|
||||
@@ -132,6 +132,11 @@ func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Move
|
||||
s.logf("%s keeps the movers revision %d of its own source %d re-pinned, %s: %d steps ended superseded, %d asked again (H-A-283, H-A-284)",
|
||||
key, half.Revision, half.Source, strings.Join(moved, ", "), len(ended), len(asks))
|
||||
s.sayRePinned(p, kept, revisions, half.Revision)
|
||||
// The re-pin says pipeline-started again with every revision its movers are kept at, so the indexer pins them (design
|
||||
// 47 H2 H-A-496 and its addendum).
|
||||
if keeps {
|
||||
s.sayStartedPipeline(p, now)
|
||||
}
|
||||
for _, st := range ended {
|
||||
if s.Say == nil {
|
||||
break
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
asks := &stepAsks{}
|
||||
var said []seat.StepEvent
|
||||
var repinned []string
|
||||
var repinned, starts []string
|
||||
var left [][]seat.StepResult
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: asks, Name: "the-runner", Now: ticking(), Revision: read,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
|
||||
@@ -77,6 +77,9 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
if ev, ok := body.(seat.RePinnedEvent); ok {
|
||||
repinned = append(repinned, fmt.Sprintf("%s %s %s %d", subject, ev.Pipeline, ev.Mover, ev.Revision))
|
||||
}
|
||||
if ev, ok := body.(seat.PipelineStarted); ok {
|
||||
starts = append(starts, fmt.Sprintf("%s %v", ev.Pipeline, ev.Kept))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Left: func(_ string, leave, drop []seat.StepResult, _ string) { left = append(left, append(leave, drop...)) }}
|
||||
@@ -133,6 +136,11 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
if kept, err := store.KeptRevisions(key); err != nil || kept["app"] != 11 {
|
||||
t.Errorf("kept-revisions = %v (%v), want app at 11", kept, err)
|
||||
}
|
||||
// design 47 H2 H-A-496 and its addendum: the start names no kept revision, the re-pin says pipeline-started again with
|
||||
// every revision its movers are kept at.
|
||||
if want := []string{key + " []", key + " [11]"}; !slices.Equal(starts, want) {
|
||||
t.Errorf("pipeline-started said %q, want %q", starts, want)
|
||||
}
|
||||
// The pin check reads the kept mover's entry at the revision it was kept at, never the pipeline's.
|
||||
if n, err := s.AskPinHeld(store, key, "app", map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26", "git.novox.be/novox/x/go": "v0.1.1"}, read); n == 0 || err != nil {
|
||||
after, _, _ := store.Get(key)
|
||||
@@ -255,6 +263,60 @@ func TestAKeptMoverWaitingOnItsRevisionSaysSo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// design 47 H2 H-A-496: a kept mover whose kept revision the catalog answers "no such object" for waits no longer: the
|
||||
// pipeline ends failed "kept revision 11 gone" at the placing, is said ended, frees its turn, and leaves its mover
|
||||
// unfinished, so it rejoins the next pipeline.
|
||||
func TestAKeptRevisionGoneEndsThePipeline(t *testing.T) {
|
||||
reg, entries, _, read := repinFixture()
|
||||
revision := func(branch string, rev int64) (Half, error) {
|
||||
if rev == 11 {
|
||||
return Half{}, fmt.Errorf("revision main.11: no such object: %w", ErrRevisionGone)
|
||||
}
|
||||
return read(branch, rev)
|
||||
}
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
var turned, ended []string
|
||||
u := memUnfinished{}
|
||||
h := &Hearer{Store: store, Name: "the-runner", Unfinished: &Unfinished{Store: u},
|
||||
Say: func(subject string, body any) error {
|
||||
if ev, ok := body.(seat.PipelineEnded); ok {
|
||||
ended = append(ended, ev.Pipeline+" "+ev.Outcome)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Turn: func(key string) { turned = append(turned, key) }}
|
||||
registered := false
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: &stepAsks{}, Name: "the-runner", Now: ticking(), Revision: revision,
|
||||
Say: func(string, any) error { return nil }, Ended: h.EndedAtPlacing,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
|
||||
key := Key("main", 10)
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: 41, Revision: 10, Entries: entries,
|
||||
Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
if n, err := s.Revised(revisionEvent(11), read); n != 0 || err != nil {
|
||||
t.Fatalf("the re-pin started %d (%v)", n, err)
|
||||
}
|
||||
registered = true
|
||||
lagging := func(string) (Half, bool) { return Half{Revision: 10, Source: 41, Entries: entries}, true }
|
||||
if _, err := s.Registered(store, reg, lagging); err != nil {
|
||||
t.Fatalf("the placing: %v", err)
|
||||
}
|
||||
p, _, err := store.Get(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Outcome != seat.PipelineOutcomeFailed || p.Ended == nil || !strings.HasPrefix(p.Why, "kept revision 11 gone") {
|
||||
t.Fatalf("the pipeline = %s, %q, ended %v: want failed with kept revision 11 gone", p.Outcome, p.Why, p.Ended)
|
||||
}
|
||||
if !slices.Equal(ended, []string{key + " failed"}) || !slices.Equal(turned, []string{key}) {
|
||||
t.Fatalf("said ended %v, turns freed %v: want %s's end said and its turn freed", ended, turned, key)
|
||||
}
|
||||
if _, held := u[UnfinishedKey("main", "app")]; !held {
|
||||
t.Fatalf("app is not left unfinished, so it rejoins no pipeline: %v", u)
|
||||
}
|
||||
}
|
||||
|
||||
// rawKept reads a memRaw as the reads do.
|
||||
type rawKept struct{ r *memRaw }
|
||||
|
||||
|
||||
@@ -347,6 +347,27 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A
|
||||
return nil, ended
|
||||
}
|
||||
|
||||
// keptGone ends p failed at its placing, "kept revision <r> gone", when the catalog no longer holds revision r a mover of
|
||||
// it is kept at (design 47 H2 H-A-496): each step not ended ends not asked, so its movers, whose steps never ran, are left
|
||||
// unfinished and rejoin the next pipeline (§786), and its end frees its turn under pipelines-at-once. It answers the steps
|
||||
// it ended, none when it had none; the pipeline's Ended says it ended.
|
||||
func keptGone(p *seat.Pipeline, r int64, err error, now time.Time) []int {
|
||||
why := fmt.Sprintf("kept revision %d gone", r)
|
||||
ended := []int{}
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
st.Outcome, st.Why, st.Ended, st.Ask, st.Shared = seat.StepResultOutcomeNotAsked, "the pipeline failed: "+why, &now, "", nil
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
}
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, fmt.Sprintf("%s (%v)", why, err), &now
|
||||
p.EndCount++
|
||||
return ended
|
||||
}
|
||||
|
||||
// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the
|
||||
// pipeline's end, through Ended; with none given it is said in the journal alone.
|
||||
func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error {
|
||||
@@ -416,18 +437,36 @@ func (s *Starter) create(key string, p seat.Pipeline) error {
|
||||
return s.Pipelines.Create(key, raw)
|
||||
}
|
||||
|
||||
// started is a pipeline's pipeline-started (§746): its key, branch, source, call, revision and movers.
|
||||
func started(p seat.Pipeline, at time.Time) seat.PipelineStarted {
|
||||
// started is a pipeline's pipeline-started (§746): its key, branch, source, call, revision and movers, and kept, every
|
||||
// catalog revision its movers are kept at (design 47 H2 H-A-496), so the indexer pins each while the pipeline runs.
|
||||
func started(p seat.Pipeline, at time.Time, kept []int64) seat.PipelineStarted {
|
||||
return seat.PipelineStarted{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call,
|
||||
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run}
|
||||
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run, Kept: kept}
|
||||
}
|
||||
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<revision> (§746).
|
||||
// keptList is every distinct revision of kept-revisions, in order (H-A-496).
|
||||
func keptList(kept map[string]int64) []int64 {
|
||||
var out []int64
|
||||
for _, r := range kept {
|
||||
if r > 0 && !slices.Contains(out, r) {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<revision> (§746), with the revisions its movers are kept at as
|
||||
// its record's kept-revisions holds them (H-A-496): a re-pin says it again with the full list (H-A-496 addendum).
|
||||
func (s *Starter) sayStartedPipeline(p seat.Pipeline, at time.Time) {
|
||||
if s.Say == nil {
|
||||
return
|
||||
}
|
||||
if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at)); err != nil {
|
||||
kept, err := keptOfStore(s.Store, p.Pipeline)
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its pipeline-started names no kept revision: %v", p.Pipeline, err)
|
||||
}
|
||||
if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at, keptList(kept))); err != nil {
|
||||
s.logf("pipeline-started of %s is not said: %v", p.Pipeline, err)
|
||||
}
|
||||
}
|
||||
@@ -445,6 +484,8 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its movers are placed as movers not kept: %v", p.Pipeline, err)
|
||||
}
|
||||
var gone int64
|
||||
var goneWhy error
|
||||
all := entries
|
||||
for i, m := range p.Movers {
|
||||
// A mover that left for a newer pipeline is placed there, never here (H-A-223, H-A-361).
|
||||
@@ -460,7 +501,13 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
entries := all
|
||||
if rev := kept[m.Module]; rev > 0 {
|
||||
var err error
|
||||
if entries, err = s.keptEntries(p.Branch, rev); err != nil {
|
||||
if entries, err = s.keptEntries(p.Branch, rev); errors.Is(err, ErrRevisionGone) {
|
||||
// A kept revision the catalog no longer holds is never read: the pipeline ends failed (H-A-496).
|
||||
if gone == 0 {
|
||||
gone, goneWhy = rev, err
|
||||
}
|
||||
continue
|
||||
} else if err != nil {
|
||||
// It waits for its kept revision to be read, said once and carried on the mover (design 47 H2 §787).
|
||||
if m.Unplaced != seat.MoverUnplacedKeptRevisionUnread || m.Revision != rev {
|
||||
s.logf("%s: %s waits for its kept revision %d to be read (%v)", p.Pipeline, m.Module, rev, err)
|
||||
@@ -520,6 +567,9 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
}
|
||||
}
|
||||
}
|
||||
if gone > 0 {
|
||||
return nil, unregistered, keptGone(p, gone, goneWhy, now), notAsked
|
||||
}
|
||||
if steps, ended := cycled(p, waits, unregistered, reg, all, now, s.Name); ended {
|
||||
return nil, unregistered, steps, notAsked
|
||||
} else {
|
||||
@@ -702,7 +752,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
}
|
||||
s.ran(key)
|
||||
s.sayNotAsked(p, notAsked)
|
||||
if len(ended) > 0 {
|
||||
if len(ended) > 0 || p.Ended != nil {
|
||||
return true, s.endedAtPlacing(p, ended)
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
@@ -878,16 +928,19 @@ func (s *Starter) placeWaiting(store Updater, due func(seat.Pipeline) bool, reg
|
||||
if err != nil {
|
||||
note(err)
|
||||
}
|
||||
if n == 0 {
|
||||
// A pipeline its placing ended (a kept revision gone, H-A-496) is said ended though it placed no step.
|
||||
if n == 0 && ended == nil {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
s.logf("%s: %d steps placed from %s (A-327)", key, n, from)
|
||||
// A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start,
|
||||
// said already, is said again harmlessly.
|
||||
s.sayStartedPipeline(p, now)
|
||||
if n > 0 {
|
||||
s.logf("%s: %d steps placed from %s (A-327)", key, n, from)
|
||||
// A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start,
|
||||
// said already, is said again harmlessly.
|
||||
s.sayStartedPipeline(p, now)
|
||||
}
|
||||
s.sayNotAsked(p, notAsked)
|
||||
if len(ended) > 0 {
|
||||
if ended != nil {
|
||||
if err := s.endedAtPlacing(p, ended); err != nil {
|
||||
note(err)
|
||||
}
|
||||
@@ -933,7 +986,7 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
|
||||
s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m)
|
||||
s.registrationMissing(m, key)
|
||||
}
|
||||
if len(p.Steps) == before {
|
||||
if len(p.Steps) == before && p.Ended == nil {
|
||||
// A mover that now waits for its kept revision, or no longer does, is written so (design 47 H2 §787).
|
||||
if unplacedMarks(p) == marks {
|
||||
return p, nil, nil, nil, 0, nil
|
||||
|
||||
@@ -27,6 +27,10 @@ const (
|
||||
IdleAfter = 24 * time.Hour
|
||||
// FullPercent is the fill of the objects past which the receiver says they fill.
|
||||
FullPercent = 80
|
||||
// TakeOverAfter is how long a transfer's last part must lie behind it before another caller opening it with the
|
||||
// same digest takes it over (design 47 H2 H-A-481): a sender restarted mid-transfer goes on, a live one is not
|
||||
// pushed aside.
|
||||
TakeOverAfter = 2 * time.Minute
|
||||
)
|
||||
|
||||
// Want is one part an opening names: its digest (sha256:<hex>) and its size.
|
||||
@@ -117,12 +121,17 @@ type Receiver[M any] struct {
|
||||
// Bound reads the cap the objects' binding carries (BusCap): the bus's own bound, which then counts over Cap.
|
||||
// Optional; an error or a zero falls back to Cap.
|
||||
Bound func() (uint64, error)
|
||||
// MaxChunk, Idle and Full override MaxReceived, IdleAfter and FullPercent when set.
|
||||
// MaxChunk, Idle, TakeOver and Full override MaxReceived, IdleAfter, TakeOverAfter and FullPercent when set.
|
||||
MaxChunk int
|
||||
Idle time.Duration
|
||||
TakeOver time.Duration
|
||||
Full uint64
|
||||
// Holds says the holder already holds a part, which then counts as held at the open and is not sent. Optional.
|
||||
Holds func(t Transfer[M], digest string) bool
|
||||
// HoldsWhole says the holder holds the file itself the transfer brings (its manifest or reference at the name and
|
||||
// version its shape names), never from its parts alone (design 47 H2 H-A-481 second addendum): a reopen that lacks
|
||||
// nothing is then dropped without a second store. Optional: nil never drops, and the reopen finishes as usual.
|
||||
HoldsWhole func(t Transfer[M]) bool
|
||||
// Check is the caller's check of a whole transfer before anything is stored; its error is the refusal's reason.
|
||||
// Optional.
|
||||
Check func(t Transfer[M]) error
|
||||
@@ -172,6 +181,13 @@ func (r *Receiver[M]) CapNow() uint64 {
|
||||
return r.Cap
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) takeOver() time.Duration {
|
||||
if r.TakeOver > 0 {
|
||||
return r.TakeOver
|
||||
}
|
||||
return TakeOverAfter
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) fullPct() uint64 {
|
||||
if r.Full > 0 {
|
||||
return r.Full
|
||||
@@ -180,8 +196,12 @@ func (r *Receiver[M]) fullPct() uint64 {
|
||||
}
|
||||
|
||||
// Open opens a transfer for opener, or opens it again: a transfer already open answers what it lacks, so a cut transfer
|
||||
// goes on from there, and only its opener may open it again. A new transfer that would pass the cap is refused; parts
|
||||
// the holder holds count as held; when nothing is lacked the transfer ends at once.
|
||||
// goes on from there. A transfer is finished or taken over, never left open (design 47 H2 H-A-481 and its addendum):
|
||||
// one opened again with nothing lacked (its finish was cut) is finished, by its opener or by another caller with the
|
||||
// same digest, once each part taken in it is read back and hashes to its digest (a part that does not is lacked again),
|
||||
// and dropped without a second store when the holder already holds the file itself (HoldsWhole); one that lacks a part is another
|
||||
// caller's only once its last part is older than TakeOver, and is refused to it before. A new transfer that would pass the cap is refused; parts the holder holds count as held; when nothing is
|
||||
// lacked the transfer ends at once, its idle bound armed before the finish, so a cut finish never leaves it open.
|
||||
func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) {
|
||||
switch {
|
||||
case strings.TrimSpace(id) == "" || strings.ContainsAny(id, "/. *>"):
|
||||
@@ -197,8 +217,33 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
if t, found, err := r.Transfers.Get(id); err != nil {
|
||||
return Received{}, err
|
||||
} else if found {
|
||||
if t.Opener != opener {
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id)
|
||||
if len(t.Lacks()) == 0 {
|
||||
if r.heldWhole(t) {
|
||||
// The cut came after the store: the holder holds the file, so it is not stored a second time.
|
||||
if _, err := r.Drop(id); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s is stored already and cannot be dropped: %w", id, err)
|
||||
}
|
||||
return Received{ID: id, Ended: true}, nil
|
||||
}
|
||||
if !r.verify(&t) {
|
||||
return r.finishArmed(t)
|
||||
}
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err)
|
||||
}
|
||||
r.tell(t.FillAtOpen)
|
||||
}
|
||||
switch {
|
||||
case len(t.Lacks()) == 0:
|
||||
return r.finishArmed(t)
|
||||
case t.Opener != opener && r.now().Sub(t.Last) < r.takeOver():
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller, whose last part came %s ago (taken over after %s)",
|
||||
opener, id, r.now().Sub(t.Last).Round(time.Second), r.takeOver())
|
||||
case t.Opener != opener:
|
||||
t.Opener = opener
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be taken over by %s: %w", id, opener, err)
|
||||
}
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
@@ -227,11 +272,53 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
}
|
||||
r.tell(used)
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
return r.finishArmed(t)
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
|
||||
// heldWhole says the holder already holds the file t brings (HoldsWhole), so a finish cut after its store stores nothing
|
||||
// again; parts held alone never say it (H-A-481 second addendum), since a publish cut before its store must still be
|
||||
// said as published or refused.
|
||||
func (r *Receiver[M]) heldWhole(t Transfer[M]) bool {
|
||||
return r.HoldsWhole != nil && r.HoldsWhole(t)
|
||||
}
|
||||
|
||||
// verify reads each part taken in t back from the objects and hashes it against its digest, as Take does at a part's
|
||||
// last chunk (H-A-481 addendum): a finish cut after its chunks were dropped reads them absent, which the runtime's
|
||||
// object read answers as empty. A part that fails has its chunks dropped and is lacked again. It answers whether any
|
||||
// part failed, t changed with it.
|
||||
func (r *Receiver[M]) verify(t *Transfer[M]) bool {
|
||||
failed := false
|
||||
for _, p := range t.Parts {
|
||||
if !t.Taken(p.Digest) || !slices.Contains(t.Held, p.Digest) {
|
||||
continue
|
||||
}
|
||||
whole, err := r.part(*t, p.Digest)
|
||||
if err == nil && "sha256:"+sum(whole) == p.Digest {
|
||||
continue
|
||||
}
|
||||
r.dropChunks(*t, p.Digest)
|
||||
delete(t.Chunks, p.Digest)
|
||||
t.Held = slices.DeleteFunc(t.Held, func(d string) bool { return d == p.Digest })
|
||||
failed = true
|
||||
}
|
||||
if failed {
|
||||
r.reanchor(t)
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
// finishArmed finishes a transfer that lacks nothing at an open, its idle bound armed first (H-A-481): a finish cut
|
||||
// by a restart leaves a kept transfer its key still drops, and the next open finishes it. A key not written fails the
|
||||
// open loudly before the finish, the transfer kept for the next open.
|
||||
func (r *Receiver[M]) finishArmed(t Transfer[M]) (Received, error) {
|
||||
if err := r.arm(t.ID); err != nil {
|
||||
return Received{}, err
|
||||
}
|
||||
return Received{ID: t.ID, Ended: true}, r.finish(t)
|
||||
}
|
||||
|
||||
// arm bounds an open transfer by its idle key. A key that cannot be written fails the open loudly (the transfer is
|
||||
// kept, so the opener's next open of it answers what is held and arms it again).
|
||||
func (r *Receiver[M]) arm(id string) error {
|
||||
|
||||
@@ -289,6 +289,106 @@ func aHolderTakesAFileInParts(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// Design 47 H2 H-A-481: a transfer is finished or taken over, never left open. A finish cut by a restart leaves the
|
||||
// transfer kept with every part held and its chunks in the objects (cut): the opener's next open, or another caller's
|
||||
// with the same digest, finishes it; a transfer that lacks a part is another caller's once its last part is older
|
||||
// than TakeOverAfter, refused to it before; and a transfer that lacks nothing at its open is armed before its finish.
|
||||
t.Run("a transfer is finished or taken over, never left open", func(t *testing.T) {
|
||||
var order []string
|
||||
pkg.Arm = func(id string, _ time.Duration) error { order = append(order, "arm "+id); return nil }
|
||||
store := pkg.Store
|
||||
defer func() { pkg.Arm, pkg.Holds, pkg.HoldsWhole, pkg.Store = nil, nil, nil, store }()
|
||||
// cut leaves c1 as a cut finish does: "whole" all chunks there, "lacked" a part short, "gone" its chunks dropped
|
||||
// before the cut, "wrong" a chunk that does not hash.
|
||||
cut := func(id, how string) {
|
||||
tr := Transfer[packageShape]{ID: id, Opener: "builder", Parts: []Want{part}, Chunks: map[string]int{part.Digest: 1},
|
||||
Held: []string{part.Digest}, Opened: now, Last: now, Shape: packageShape{"m", "1"}}
|
||||
switch how {
|
||||
case "whole":
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = blob
|
||||
case "lacked":
|
||||
tr.Held = nil
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = blob[:1000]
|
||||
case "wrong":
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = []byte("other")
|
||||
}
|
||||
if err := pkg.Transfers.Put(tr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name, opener, how string
|
||||
after, takeOver time.Duration
|
||||
storeHolds string // "parts": Holds answers every part; "file": HoldsWhole answers the file
|
||||
want string // "stored", "dropped", "lacked again", "lacks" (taken over) or the refusal's words
|
||||
}{
|
||||
{name: "the opener opens a cut finish again", opener: "builder", how: "whole", want: "stored"},
|
||||
{name: "another caller opens a cut finish", opener: "other", how: "whole", want: "stored"},
|
||||
{name: "a cut after the chunks were dropped", opener: "builder", how: "gone", want: "lacked again"},
|
||||
{name: "a cut finish whose chunk does not hash", opener: "builder", how: "wrong", want: "lacked again"},
|
||||
{name: "a cut after the store: the holder holds the file", opener: "builder", how: "whole", storeHolds: "file", want: "dropped"},
|
||||
{name: "a cut before the store: the holder holds the parts, not the file", opener: "builder", how: "whole", storeHolds: "parts", want: "stored"},
|
||||
{name: "another caller while the opener sends", opener: "other", how: "lacked", after: time.Minute, want: "another caller, whose last part came 1m0s ago"},
|
||||
{name: "another caller once the last part is two minutes old", opener: "other", how: "lacked", after: TakeOverAfter, want: "lacks"},
|
||||
{name: "the receiver's own takeover bound", opener: "other", how: "lacked", after: time.Minute, takeOver: 30 * time.Second, want: "lacks"},
|
||||
} {
|
||||
pkgLog = nil
|
||||
pkg.TakeOver = c.takeOver
|
||||
pkg.Holds, pkg.HoldsWhole = nil, nil
|
||||
switch c.storeHolds {
|
||||
case "parts":
|
||||
pkg.Holds = func(Transfer[packageShape], string) bool { return true }
|
||||
case "file":
|
||||
pkg.HoldsWhole = func(tr Transfer[packageShape]) bool { return tr.Shape == packageShape{"m", "1"} }
|
||||
}
|
||||
cut("c1", c.how)
|
||||
now = now.Add(c.after)
|
||||
got, err := pkg.Open("c1", c.opener, []Want{part}, packageShape{"m", "1"})
|
||||
kept, open, _ := pkg.Transfers.Get("c1")
|
||||
switch c.want {
|
||||
case "stored":
|
||||
if err != nil || !got.Ended || open || len(pkgLog) != 1 || pkgLog[0].outcome != "stored" || string(pkgLog[0].parts[part.Digest]) != string(blob) {
|
||||
t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it finished and stored", c.name, got, err, open, pkgLog)
|
||||
}
|
||||
case "dropped":
|
||||
if err != nil || !got.Ended || open || len(pkgLog) != 0 {
|
||||
t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it dropped, not stored again", c.name, got, err, open, pkgLog)
|
||||
}
|
||||
case "lacked again":
|
||||
if err != nil || got.Ended || !open || len(pkgLog) != 0 || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) || kept.Taken(part.Digest) {
|
||||
t.Errorf("%s: open = %+v, %v, log %+v, kept %+v; want the part lacked again, nothing stored", c.name, got, err, pkgLog, kept)
|
||||
}
|
||||
_, _ = pkg.Drop("c1")
|
||||
case "lacks":
|
||||
if err != nil || got.Ended || !open || kept.Opener != c.opener || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) {
|
||||
t.Errorf("%s: open = %+v, %v, opener %q; want it taken over by %s, lacking the part", c.name, got, err, kept.Opener, c.opener)
|
||||
}
|
||||
if r, err := pkg.Take("c1", c.opener, part.Digest, 1, true, blob[1000:], nil); err != nil || !r.Ended {
|
||||
t.Errorf("%s: the new opener's last chunk = %+v, %v", c.name, r, err)
|
||||
}
|
||||
default:
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) || !open || kept.Opener != "builder" {
|
||||
t.Errorf("%s: open = %v, opener %q; want refused with %q, the transfer kept", c.name, err, kept.Opener, c.want)
|
||||
}
|
||||
_, _ = pkg.Drop("c1")
|
||||
}
|
||||
if len(objects) != 0 {
|
||||
t.Errorf("%s: %d objects left", c.name, len(objects))
|
||||
}
|
||||
}
|
||||
pkg.TakeOver = 0
|
||||
order = nil
|
||||
pkg.Holds = func(Transfer[packageShape], string) bool { return true }
|
||||
pkg.Store = func(tr Transfer[packageShape], _ map[string][]byte) error {
|
||||
order = append(order, "store "+tr.ID)
|
||||
return nil
|
||||
}
|
||||
if got, err := pkg.Open("c2", "builder", []Want{part}, packageShape{"m", "1"}); err != nil || !got.Ended ||
|
||||
!reflect.DeepEqual(order, []string{"arm c2", "store c2"}) {
|
||||
t.Errorf("a transfer lacking nothing at its open = %+v, %v, %v; want its idle bound armed before its finish", got, err, order)
|
||||
}
|
||||
})
|
||||
|
||||
// An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the
|
||||
// rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the
|
||||
// binding carries (Bound) counts over Cap.
|
||||
|
||||
@@ -84,7 +84,14 @@ func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, erro
|
||||
if ask.Build != nil {
|
||||
job := *ask.Build
|
||||
job.Fetch = withPackages(job.Fetch, ask.Packages)
|
||||
return steps.build(ctx, job, ask.Version)
|
||||
out, err := steps.build(ctx, job, ask.Version)
|
||||
var dec *Declining
|
||||
if err != nil && !sdkstep.IsPassing(err) && !errors.As(err, &dec) && ctx.Err() == nil {
|
||||
// A failed build keeps its error text, as the build module bounded it, as the ask's output object, header and
|
||||
// text as a test step's, so the verb output answers why after the fact (design 47 H2 H-A-482 and its addendum (a)).
|
||||
steps.keepRunOutput(ask.ID, err.Error(), nil)
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
if ask.Step != nil || ask.Install != nil {
|
||||
return steps.runStep(ctx, ask)
|
||||
|
||||
@@ -117,6 +117,23 @@ imports {"module":"modules/m","workspace":"/w/1"}`
|
||||
if !slices.Equal(answer.Imports, []string{"git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/health"}) {
|
||||
t.Errorf("the answer's imports are %v", answer.Imports)
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-482 addendum (a): a failed build keeps its error text, as the build module bounded it, as the ask's
|
||||
// output object, which the verb output answers.
|
||||
outputs := fakeOutputs{m: map[string][]byte{}}
|
||||
steps.Outputs, steps.Node = outputs, "ace"
|
||||
steps.Build = func(context.Context, map[string]any) (any, error) {
|
||||
return nil, errors.New("esbuild for code: status 1 | ✘ [ERROR] Could not resolve \"./missing\" … 812 bytes left out … main.js:1:1)")
|
||||
}
|
||||
failed := ask(sdkstep.BuildJob{Module: "m", Path: "modules/m", Fetch: &sdkstep.Fetch{Repository: "novox/x", Commit: runCommit},
|
||||
Artifacts: []sdkstep.AskArtifact{{Artifact: "bundle"}}})
|
||||
if _, err := RunAsk(context.Background(), steps, failed); err == nil {
|
||||
t.Fatal("a failed build answered no error")
|
||||
}
|
||||
got2, err := readOutput(outputs, "ace", failed.ID, nil)
|
||||
if err != nil || !strings.Contains(got2.Output, `bundle: esbuild for code: status 1 | ✘ [ERROR] Could not resolve "./missing"`) || got2.Size != len(got2.Output) {
|
||||
t.Errorf("the failed build's output = %+v, %v; want its error text kept", got2, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build ask for another toolchain is declined (ADR 0584 clause 2), so the node sits out and the asker puts it again.
|
||||
|
||||
@@ -125,7 +125,8 @@ func EndedHere(node string, keys func() ([]string, error), get func(key string)
|
||||
// readOutput answers the verb output for ask (design 47 H2 §791, H-A-412): from the object `<node>.<ask>`, the text's last
|
||||
// OutputTail bytes, its size and failing tests from the header; with no object, no longer kept when this node's steps key
|
||||
// shows the ask ended here more than OutputsMaxAge ago, and refused otherwise, naming why (an ask never run here, or one
|
||||
// whose output this node did not keep).
|
||||
// that kept no output). An empty object is no object (H-A-482): the runtime's object read answers an absent object as
|
||||
// empty data without an error, which is not a header fault.
|
||||
func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.Time, bool, error)) (sdkstep.OutputAnswer, error) {
|
||||
if o == nil {
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("this module keeps no %s store", OutputsStore)
|
||||
@@ -135,11 +136,14 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.
|
||||
return sdkstep.OutputAnswer{}, err
|
||||
}
|
||||
got, gerr := o.Get(name)
|
||||
if gerr == nil {
|
||||
if gerr == nil && len(got.Data) > 0 {
|
||||
return outputOf(ask, got.Data)
|
||||
}
|
||||
if gerr == nil {
|
||||
gerr = fmt.Errorf("the object %s in the %s store is empty", name, OutputsStore)
|
||||
}
|
||||
if ended == nil {
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the output %s in the %s store: %w", name, OutputsStore, gerr)
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s kept no output: %w", ask, gerr)
|
||||
}
|
||||
at, here, err := ended(ask)
|
||||
switch {
|
||||
@@ -150,7 +154,7 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.
|
||||
case !at.IsZero() && now().Sub(at) > OutputsMaxAge:
|
||||
return sdkstep.OutputAnswer{Ask: ask, NoLongerKept: true}, nil
|
||||
default:
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and its output is not in the %s store: %w", ask, node, OutputsStore, gerr)
|
||||
return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and the ask kept no output: %w", ask, node, gerr)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -64,7 +64,9 @@ func TestOutputsKeptAreRead(t *testing.T) {
|
||||
return raw
|
||||
}
|
||||
steps := map[string]json.RawMessage{"mesh-delivery-pipeline.ace.a2": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a3": step(at.Add(-time.Hour)),
|
||||
"mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour))}
|
||||
"mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a6": step(at.Add(-time.Hour))}
|
||||
// H-A-482: the runtime's object read answers an absent object as empty data, no error; that is no output, not a header fault.
|
||||
out.m["ace.a6"] = []byte{}
|
||||
keys := func() ([]string, error) {
|
||||
return slices.Collect(func(y func(string) bool) {
|
||||
for k := range steps {
|
||||
@@ -82,7 +84,8 @@ func TestOutputsKeptAreRead(t *testing.T) {
|
||||
}{
|
||||
{ask: "a1", want: sdkstep.OutputAnswer{Ask: "a1", Output: long[len(long)-OutputTail:], Size: len(long), Failing: failing}},
|
||||
{ask: "a2", want: sdkstep.OutputAnswer{Ask: "a2", NoLongerKept: true}},
|
||||
{ask: "a3", fail: "ran on ace, and its output is not in the outputs store"},
|
||||
{ask: "a3", fail: "ran on ace, and the ask kept no output"},
|
||||
{ask: "a6", fail: "ran on ace, and the ask kept no output: the object ace.a6 in the outputs store is empty"},
|
||||
{ask: "a4", fail: "never ran on ace"},
|
||||
{ask: "a5", fail: "never ran on ace"},
|
||||
{fail: "asked for no ask"},
|
||||
|
||||
@@ -267,8 +267,9 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste
|
||||
return nil
|
||||
}
|
||||
|
||||
// keepRunOutput keeps a test step's level text as the run's output `<node>.<ask id>` in the outputs objects, with the
|
||||
// header of the failing tests and their words from the level's results (design 47 H2 §791, H-A-412 (1)). A holder that
|
||||
// keepRunOutput keeps a test step's level text, or a failed build's error text (H-A-482), as the run's output `<node>.<ask
|
||||
// id>` in the outputs objects, with the header of the failing tests and their words from the level's results (design 47
|
||||
// H2 §791, H-A-412 (1)). A holder that
|
||||
// keeps no outputs, or a write that fails, is said in the journal: the step's end stands on the results.
|
||||
func (s AskSteps) keepRunOutput(ask, text string, failed []sdkstep.Failure) {
|
||||
var failing []sdkstep.FailingTest
|
||||
|
||||
@@ -5,13 +5,15 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The part's own membership names the stream running among those it hears, and hearRunning binds the part's durable on it:
|
||||
// The part's own membership names the stream running among those it hears, and hearRunning binds the part's durable on it
|
||||
// (and EVENTS, which the hub reads, H-A-472 seventh addendum):
|
||||
// the loop that opens the other streams leaves it alone, so no second reader takes messages of that one durable (design 47 G
|
||||
// §556, A-19; the whole-part integration test lost the running event to the second reader at random).
|
||||
func TestTheLoopOfOwnHearsLeavesTheStreamsADedicatedHearingOwns(t *testing.T) {
|
||||
hears := []membership.HeardIssued{{Stream: nodeengine.StreamRunning, Consumer: "a_mesh-grants"}, {Stream: "mod-c_feed", Consumer: "a_mesh-grants"}, {Stream: "opened_one"}}
|
||||
hears := []membership.HeardIssued{{Stream: nodeengine.StreamRunning, Consumer: "a_mesh-grants"}, {Stream: "mod-c_feed", Consumer: "a_mesh-grants"}, {Stream: "opened_one"}, {Stream: process.EventsStream, Consumer: "a_mesh-grants"}}
|
||||
got := hearsToOpen(hears, map[string]bool{"opened_one": true})
|
||||
if len(got) != 1 || got[0].Stream != "mod-c_feed" {
|
||||
t.Errorf("the streams opened are %+v, want mod-c_feed alone", got)
|
||||
|
||||
@@ -313,11 +313,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// hearsToOpen are the streams of the part's own membership no hearing has opened yet. A stream another hearing of the part
|
||||
// binds its durable on (running: hearRunning, which reads it whole at the start before it follows) is left to that hearing:
|
||||
// two readers of one durable each take the messages the other does not, and the event a reader of its own never hears is lost
|
||||
// to it (the integration test of the whole part met it as a state running that stayed empty, 2026-11-12).
|
||||
// to it (the integration test of the whole part met it as a state running that stayed empty, 2026-11-12). So is EVENTS, which
|
||||
// the membership names like any stream since it carries the consumer's spec (design 47 H2, H-A-472 seventh addendum): the
|
||||
// hub reads the part's durable there.
|
||||
func hearsToOpen(hears []membership.HeardIssued, opened map[string]bool) []membership.HeardIssued {
|
||||
var out []membership.HeardIssued
|
||||
for _, h := range hears {
|
||||
if !opened[h.Stream] && h.Stream != nodeengine.StreamRunning {
|
||||
if !opened[h.Stream] && h.Stream != nodeengine.StreamRunning && h.Stream != process.EventsStream {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3157,6 +3157,24 @@ func TestGrants(t *testing.T) {
|
||||
if got := slices.Sorted(maps.Keys(b.own[grants.StateMemberships])); !slices.Equal(got, []string{"a.mod-x", "a.nats", "b.mod-y"}) {
|
||||
t.Errorf("memberships %v", got)
|
||||
}
|
||||
// Design 47 H2, H-A-472 ninth addendum: a module no longer assigned on its node has its membership withdrawn, so the
|
||||
// tool runner there removes its twins; a pass whose read failed withdraws nothing, and a bindings key is no membership.
|
||||
b.mu.Lock()
|
||||
b.own[grants.StateMemberships][grants.BoundKey("mod-y")] = []byte(`{}`)
|
||||
b.unassign("b", "mod-y")
|
||||
b.refused[grants.ReadHave] = true
|
||||
b.mu.Unlock()
|
||||
p.Compose()
|
||||
if _, _, found, _ := b.ownGet(grants.StateMemberships, "b.mod-y"); !found {
|
||||
t.Error("a pass whose read failed withdrew b.mod-y")
|
||||
}
|
||||
b.mu.Lock()
|
||||
delete(b.refused, grants.ReadHave)
|
||||
b.mu.Unlock()
|
||||
p.Compose()
|
||||
if got := slices.Sorted(maps.Keys(b.own[grants.StateMemberships])); !slices.Equal(got, []string{"a.mod-x", "a.nats", grants.BoundKey("mod-y")}) {
|
||||
t.Errorf("memberships once mod-y left b: %v", got)
|
||||
}
|
||||
for _, r := range grants.Reads {
|
||||
if strings.HasPrefix(r, "mesh-controller.") {
|
||||
t.Errorf("the part reads %s", r)
|
||||
@@ -3774,15 +3792,17 @@ func TestGrants(t *testing.T) {
|
||||
p := partOn(b)
|
||||
p.Consumers = b
|
||||
p.Compose()
|
||||
// The runner names no process of its own, so its consumers are its twins alone (H-A-472 second addendum).
|
||||
// The runner names no process of its own, so its consumers are its twins alone (H-A-472 second addendum), which the
|
||||
// runner creates itself from its membership's spec, granted their create by name (fifth to seventh addenda).
|
||||
for _, c := range []string{"INDEXER_VIEWS.a_node-tools_tools", "ASSIGNER_VIEWS.a_node-tools_tools"} {
|
||||
if !slices.Contains(b.consumerCreates, c) {
|
||||
t.Errorf("the tool runner's consumer %s was not made: %v", c, b.consumerCreates)
|
||||
if slices.Contains(b.consumerCreates, c) {
|
||||
t.Errorf("the part made the tool runner's twin %s: %v", c, b.consumerCreates)
|
||||
}
|
||||
}
|
||||
users := shadowOf(t, b)
|
||||
runner := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "a.node-tools" })]
|
||||
for _, s := range []string{"$JS.API.CONSUMER.MSG.NEXT.INDEXER_VIEWS.a_node-tools", "$KV.node-tools_findings.>"} {
|
||||
for _, s := range []string{"$JS.API.CONSUMER.MSG.NEXT.INDEXER_VIEWS.a_node-tools", "$KV.node-tools_findings.>",
|
||||
"$JS.API.CONSUMER.CREATE.INDEXER_VIEWS.a_node-tools_tools", "$JS.API.CONSUMER.DELETE.ASSIGNER_VIEWS.a_node-tools_tools"} {
|
||||
if !slices.ContainsFunc(runner.Publish, func(g string) bool { return buscompose.SubjectsOverlap(g, s) }) {
|
||||
t.Errorf("the tool runner may not publish %s: %v", s, runner.Publish)
|
||||
}
|
||||
@@ -4155,7 +4175,7 @@ func TestGrants(t *testing.T) {
|
||||
p.Consumers = b
|
||||
b.consumers = map[string][]map[string]any{"EVENTS": {{"name": "b_other", "filters": []string{"x"}, "max_deliver": 5}}}
|
||||
res := p.Compose()
|
||||
if res.Consumers == nil || !slices.Equal(res.Consumers.Created, []string{"EVENTS.b_mod-c", "EVENTS.b_mod-c_tools"}) {
|
||||
if res.Consumers == nil || !slices.Equal(res.Consumers.Created, []string{"EVENTS.b_mod-c"}) {
|
||||
t.Fatalf("with no lease key: %+v", res.Consumers)
|
||||
}
|
||||
var made map[string]any
|
||||
@@ -4168,7 +4188,7 @@ func TestGrants(t *testing.T) {
|
||||
t.Errorf("b_mod-c was made as %v, not as the controller makes it", made)
|
||||
}
|
||||
res = p.Compose()
|
||||
if len(res.Consumers.Created) != 0 || len(res.Consumers.Missing) != 0 || len(b.consumerCreates) != 2 {
|
||||
if len(res.Consumers.Created) != 0 || len(res.Consumers.Missing) != 0 || len(b.consumerCreates) != 1 {
|
||||
t.Errorf("a consumer the bus holds was asked again: %+v, %v", res.Consumers, b.consumerCreates)
|
||||
}
|
||||
// A held consumer that differs is never changed by a pass: it is raised as a condition naming it, and brought to
|
||||
@@ -4210,7 +4230,8 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
// 1126 (the self-check's bus-consumer probe ended): a consumer the part owns past the bound behind its stream's
|
||||
// head at two passes running raises mesh-grants.consumer-behind naming it; at the bound or under, it is cleared.
|
||||
made["pending"] = grants.BehindBound + 1
|
||||
// Read once, no pull now, far behind: behind (H-A-472 eleventh addendum), never an orphan.
|
||||
made["pending"], made["last_delivered"] = grants.BehindBound+1, "2026-10-20T21:00:00Z"
|
||||
if res = p.Compose(); len(res.Consumers.Behind) != 0 {
|
||||
t.Errorf("one look at a burst was said far behind: %+v", res.Consumers)
|
||||
}
|
||||
@@ -4233,12 +4254,9 @@ func TestGrants(t *testing.T) {
|
||||
// Design 47 H2, H-A-472 (2): a durable far behind with no pulls waiting, whose twin on the stream (b_mod-c_tools) has
|
||||
// pulls waiting and delivered within the day, is an orphan: left out of the behind check, named once in
|
||||
// mesh-grants.durable-orphaned, and cleared when it is gone; with its twin not current, it is behind again.
|
||||
var twin map[string]any
|
||||
for _, c := range b.consumers["EVENTS"] {
|
||||
if c["name"] == "b_mod-c_tools" {
|
||||
twin = c
|
||||
}
|
||||
}
|
||||
// The twin is the tool runner's, made by it from the membership's spec (the fifth addendum), never by the part.
|
||||
twin := map[string]any{"name": "b_mod-c_tools", "filters": []string{"mesh.mod.mod-y.event.done"}, "max_deliver": 5}
|
||||
b.consumers["EVENTS"] = append(b.consumers["EVENTS"], twin)
|
||||
twin["waiting_pulls"], twin["last_delivered"] = 1, "2026-10-25T21:00:00.5Z"
|
||||
made["pending"] = grants.BehindBound + 1
|
||||
p.Compose()
|
||||
@@ -4282,13 +4300,78 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
made["pending"] = grants.BehindBound
|
||||
p.Compose()
|
||||
// H-A-472 second addendum: a module with an account of its own but no process resource with no bus of its own has
|
||||
// no program reading b_<module>, so only the tool runner's twin is made.
|
||||
// H-A-472 second and fifth addenda: a module with an account of its own but no process resource with no bus of its own
|
||||
// has no program reading b_<module>, and the tool runner makes its own twin, so the part makes neither.
|
||||
b.manifestOf("mod-t", `{"module":"mod-t","version":"1","own-secrets":{"broker":"/var/lib/t/broker"},"consumes":["mod-y.done"]}`)
|
||||
b.assign("b", "mod-t")
|
||||
if res = p.Compose(); !slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t_tools") || slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t") {
|
||||
if res = p.Compose(); slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t_tools") || slices.Contains(res.Consumers.Created, "EVENTS.b_mod-t") {
|
||||
t.Errorf("a module with no program of its own: created %v", res.Consumers.Created)
|
||||
}
|
||||
// The tenth and eleventh addenda: every held durable no pass composes is still judged, both twins of a module.
|
||||
named := func(list []string, key string) bool {
|
||||
return slices.ContainsFunc(list, func(s string) bool { return s == key || strings.HasPrefix(s, key+":") })
|
||||
}
|
||||
lastOrphans := func() string {
|
||||
if len(orphaned) == 0 || orphaned[len(orphaned)-1].State != occ.StateRaised {
|
||||
return ""
|
||||
}
|
||||
return orphaned[len(orphaned)-1].Explanation
|
||||
}
|
||||
// b_mod-c_tools, no pull waiting while the program's b_mod-c is current: an orphan by (2).
|
||||
twin["waiting_pulls"], twin["pending"] = 0, grants.BehindBound+1
|
||||
made["waiting_pulls"], made["last_delivered"], made["pending"] = 1, "2026-10-25T21:00:00.5Z", 0
|
||||
// b_mod-t_tools, never read (no pull, no delivery ever), mod-t having no program: an orphan, never behind.
|
||||
tTools := map[string]any{"name": "b_mod-t_tools", "filters": []string{"mesh.mod.mod-y.event.done"}, "max_deliver": 5,
|
||||
"pending": grants.BehindBound + 1}
|
||||
b.consumers["EVENTS"] = append(b.consumers["EVENTS"], tTools)
|
||||
p.Compose()
|
||||
res = p.Compose()
|
||||
if named(res.Consumers.Behind, "EVENTS.b_mod-t_tools") || named(res.Consumers.Behind, "EVENTS.b_mod-c_tools") ||
|
||||
!strings.Contains(lastOrphans(), "EVENTS.b_mod-c_tools") || !strings.Contains(lastOrphans(), "EVENTS.b_mod-t_tools") {
|
||||
t.Errorf("the held twins: behind %v, orphaned said %q", res.Consumers.Behind, lastOrphans())
|
||||
}
|
||||
// Read once, no pull now, far behind, with no current twin: behind.
|
||||
tTools["last_delivered"] = "2026-10-20T21:00:00Z"
|
||||
p.Compose()
|
||||
if res = p.Compose(); !named(res.Consumers.Behind, "EVENTS.b_mod-t_tools") || strings.Contains(lastOrphans(), "EVENTS.b_mod-t_tools") {
|
||||
t.Errorf("a twin read once and lagging: behind %v, orphaned said %q", res.Consumers.Behind, lastOrphans())
|
||||
}
|
||||
// The plain b_mod-t, which no pass composes (mod-t's program left), far behind with no pull beside a current
|
||||
// b_mod-t_tools: an orphan by (2).
|
||||
tTools["waiting_pulls"], tTools["last_delivered"], tTools["pending"] = 1, "2026-10-25T21:00:00.5Z", 0
|
||||
b.consumers["EVENTS"] = append(b.consumers["EVENTS"], map[string]any{"name": "b_mod-t", "filters": []string{"mesh.mod.mod-y.event.done"},
|
||||
"max_deliver": 5, "pending": grants.BehindBound + 1, "last_delivered": "2026-10-20T21:00:00Z"})
|
||||
p.Compose()
|
||||
if res = p.Compose(); named(res.Consumers.Behind, "EVENTS.b_mod-t") || !strings.Contains(lastOrphans(), "EVENTS.b_mod-t,") &&
|
||||
!strings.HasSuffix(lastOrphans(), "EVENTS.b_mod-t") {
|
||||
t.Errorf("the plain durable of a module whose program left: behind %v, orphaned said %q", res.Consumers.Behind, lastOrphans())
|
||||
}
|
||||
// A stream only twins are held on (no pass composes a consumer there): mod-s keeps the stream feed, which the
|
||||
// program-less mod-u hears; its never-read twin there is judged and named.
|
||||
b.manifestOf("mod-s", `{"module":"mod-s","version":"1","emits":["fed"],"streams":[{"name":"feed","events":["fed"],"max-age-days":2,"max-mib":5}]}`)
|
||||
b.manifestOf("mod-u", `{"module":"mod-u","version":"1","own-secrets":{"broker":"/var/lib/u/broker"},"consumes":["mod-s.fed"]}`)
|
||||
b.assign("b", "mod-s")
|
||||
b.assign("b", "mod-u")
|
||||
p.Compose()
|
||||
var feed string
|
||||
raw, _, _, _ := b.ownGet(grants.StateMemberships, "b.mod-u")
|
||||
if bare, err := membership.BareGrantsMembership(raw); err == nil {
|
||||
var m membership.Membership
|
||||
_ = json.Unmarshal(bare, &m)
|
||||
for _, h := range m.Hears {
|
||||
if h.Stream != "EVENTS" {
|
||||
feed = h.Stream
|
||||
}
|
||||
}
|
||||
}
|
||||
if feed == "" {
|
||||
t.Fatalf("mod-u's membership hears no stream of mod-s: %s", raw)
|
||||
}
|
||||
b.consumers[feed] = []map[string]any{{"name": "b_mod-u_tools", "filters": []string{"x"}, "max_deliver": 5, "pending": grants.BehindBound + 1}}
|
||||
p.Compose()
|
||||
if res = p.Compose(); !strings.Contains(lastOrphans(), feed+".b_mod-u_tools") {
|
||||
t.Errorf("a twin on a stream no pass composes a consumer on: orphaned said %q (unread %v)", lastOrphans(), res.Consumers.Unread)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/AC13 (novox/hq ADR 0561 §7): after take-over a list in which more than one loaded user's hash changes, with no
|
||||
|
||||
@@ -640,9 +640,9 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
|
||||
MayPublish(perms, "$JS.API.CONSUMER.DELETE.EVENTS.a_m") || MayPublish(perms, "$JS.API.CONSUMER.CREATE.EVENTS.a_m.mesh.mod.x.event.happened") {
|
||||
t.Errorf("the grants part creates consumers as %v", u.CreatesConsumers)
|
||||
}
|
||||
// Design 47 H2, H-A-472 second addendum: the program's consumer only for a module with a program of its own; the
|
||||
// tool runner's twin for every consuming module.
|
||||
for durable, want := range map[string]bool{"a_m": true, "a_m_tools": true, "a_tooled": false, "a_tooled_tools": true} {
|
||||
// Design 47 H2, H-A-472 second and fifth addenda: the program's consumer only for a module with a program of its own;
|
||||
// the tool runner's twin never, which the runner creates itself from the membership's spec.
|
||||
for durable, want := range map[string]bool{"a_m": true, "a_m_tools": false, "a_tooled": false, "a_tooled_tools": false} {
|
||||
if MayPublish(perms, "$JS.API.CONSUMER.CREATE.EVENTS."+durable) != want {
|
||||
t.Errorf("the grants part creates EVENTS.%s: %v, want %v (%v)", durable, !want, want, u.CreatesConsumers)
|
||||
}
|
||||
|
||||
@@ -163,6 +163,14 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
||||
}, true
|
||||
}
|
||||
|
||||
// carriedPrincipal is a module the tool runner carries as the consumers it reads are composed from: the one form ConsumersOf
|
||||
// composes a carried module's consumers and HeardIssuedFor their spec from, so the two cannot differ.
|
||||
func carriedPrincipal(node string, d Declared) Principal {
|
||||
return Principal{Kind: KindModule, Node: node, Module: d.Module, HasProgram: d.HasProgram, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||
Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues}
|
||||
}
|
||||
|
||||
// ModuleConsumer is one module's durable consumer, with the module and node it is for.
|
||||
type ModuleConsumer struct {
|
||||
Node, Module string
|
||||
@@ -191,8 +199,8 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
}
|
||||
seen[key] = true
|
||||
if needed {
|
||||
// Each with the tool runner's twin beside it (A-397), the program's only for a module with a program of its
|
||||
// own (H-A-472 second addendum).
|
||||
// The program's only, for a module with a program of its own (H-A-472 second addendum); the tool runner's twin is
|
||||
// the runner's to make from the membership's spec (H-A-472 fifth to seventh addenda).
|
||||
for _, tc := range twinsOf(c, p.HasProgram) {
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: tc})
|
||||
}
|
||||
@@ -244,9 +252,7 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
// 0436 §1, §2, §6), the same reason its ordinary consumer is (ADR 0198): mesh-mail has no
|
||||
// account of its own (NoAccount) and is carried here, but its process does, and the consumer
|
||||
// it reads through must exist and survive PruneHeard whether or not the module itself speaks.
|
||||
add(Principal{Kind: KindModule, Node: p.Node, Module: d.Module, HasProgram: d.HasProgram, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||
Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues})
|
||||
add(carriedPrincipal(p.Node, d))
|
||||
}
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
@@ -254,39 +255,56 @@ func heardConsumer(p Principal, h Heard) Consumer {
|
||||
const ToolsSuffix = "_tools"
|
||||
|
||||
// readerGrants are what a reader publishes to ask about, pull from and acknowledge a durable consumer on a stream: for the
|
||||
// module's program's name and for its tool runner's twin (A-397).
|
||||
// module's program's name and for its tool runner's twin (A-397); and the create and delete of the twin by its name alone,
|
||||
// which the tool runner makes from the membership's spec and removes when the module leaves the node (design 47 H2, H-A-472
|
||||
// fifth to seventh addenda). Every filter goes in FilterSubjects, so the create is on the name, never on a filter.
|
||||
func readerGrants(stream, durable string) []string {
|
||||
var out []string
|
||||
for _, d := range []string{durable, durable + ToolsSuffix} {
|
||||
out = append(out, "$JS.API.CONSUMER.INFO."+stream+"."+d, "$JS.API.CONSUMER.MSG.NEXT."+stream+"."+d, "$JS.ACK."+stream+"."+d+".>")
|
||||
}
|
||||
return out
|
||||
twin := durable + ToolsSuffix
|
||||
return append(out, "$JS.API.CONSUMER.CREATE."+stream+"."+twin, "$JS.API.CONSUMER.DELETE."+stream+"."+twin)
|
||||
}
|
||||
|
||||
// withToolsTwin is a module's consumer and, beside it, the tool runner's twin of it: the same filters under its own name (A-397).
|
||||
func withToolsTwin(c Consumer) []Consumer {
|
||||
// toolsTwin is the tool runner's twin of a module's consumer c: the same filters under its own name (A-397), the events
|
||||
// given up on and delivered again to one consumer alone named by the twin itself.
|
||||
func toolsTwin(c Consumer) Consumer {
|
||||
twin := c
|
||||
twin.Name += ToolsSuffix
|
||||
twin.Filters = append([]string(nil), c.Filters...)
|
||||
// The events given up on and delivered again to one consumer alone are named by it: the twin's are its own.
|
||||
for i, f := range twin.Filters {
|
||||
if f == AgainFilter(c.Name) {
|
||||
twin.Filters[i] = AgainFilter(twin.Name)
|
||||
}
|
||||
}
|
||||
twin.Why = c.Why + "; the tool runner's own, beside the program's (lead answer A-397)"
|
||||
return []Consumer{c, twin}
|
||||
return twin
|
||||
}
|
||||
|
||||
// twinsOf are the consumers composed of a module's consumer c (design 47 H2, H-A-472 addendum and second addendum): the
|
||||
// tool runner's twin <name>_tools for every consuming module, and c itself, the program's, only for a module with a
|
||||
// program of its own (Declared.HasProgram), so no pass makes a durable nobody reads.
|
||||
// twinsOf are the consumers the grants part composes of a module's consumer c (design 47 H2, H-A-472 addendum, second and
|
||||
// fifth addenda): c itself, the program's, only for a module with a program of its own (Declared.HasProgram), and never the
|
||||
// tool runner's twin <name>_tools, which the runner creates from the membership's spec at a child's first subscribe.
|
||||
func twinsOf(c Consumer, hasProgram bool) []Consumer {
|
||||
both := withToolsTwin(c)
|
||||
if hasProgram {
|
||||
return both
|
||||
return []Consumer{c}
|
||||
}
|
||||
return both[1:]
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardSpec is a heard entry of consumer c's twin: its stream, the module's consumer name the twin is named from, and the
|
||||
// twin's whole spec (H-A-472 seventh addendum).
|
||||
func heardSpec(c Consumer) HeardIssued {
|
||||
twin := toolsTwin(c)
|
||||
policy := membership.DeliverAll
|
||||
switch {
|
||||
case twin.LastPerSubject:
|
||||
policy = membership.DeliverLastPerSubject
|
||||
case twin.FromNow:
|
||||
policy = membership.DeliverFromNow
|
||||
}
|
||||
return HeardIssued{Stream: c.Stream, Consumer: c.Name, Filters: twin.Filters, AckWaitSeconds: twin.AckWaitSeconds,
|
||||
MaxDeliver: twin.MaxDeliver, MaxAckPending: twin.MaxAckPending, DeliverPolicy: policy}
|
||||
}
|
||||
|
||||
// heardGrants are what a principal publishes to read and acknowledge the consumers it hears module streams
|
||||
@@ -345,15 +363,21 @@ func streamsIssuedFor(d Declared) []StreamIssued {
|
||||
return out
|
||||
}
|
||||
|
||||
// HeardIssuedFor is every module stream an assignment hears, as its membership names it.
|
||||
// HeardIssuedFor is every stream an assignment hears, as its membership names it: EVENTS like any stream, each entry with
|
||||
// its consumer's whole spec as the tool runner's twin of it is composed (design 47 H2, H-A-472 seventh addendum), so the
|
||||
// runner computes no name and no filter and the grants part stays the one composer of filters.
|
||||
func HeardIssuedFor(node string, d Declared) []HeardIssued {
|
||||
var out []HeardIssued
|
||||
p := carriedPrincipal(node, d)
|
||||
if c, needed := ConsumerFor(p); needed {
|
||||
out = append(out, heardSpec(c))
|
||||
}
|
||||
// A restricted stream's bind word only for a module of its own account (novox/hq ADR 0376 §1).
|
||||
for _, h := range ComposedHears(d) {
|
||||
for _, h := range p.Hears {
|
||||
if len(h.Filters) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, HeardIssued{Stream: h.Stream, Consumer: node + "_" + d.Module})
|
||||
out = append(out, heardSpec(heardConsumer(p, h)))
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Stream < out[j].Stream })
|
||||
return out
|
||||
|
||||
@@ -38,7 +38,7 @@ accounts {
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "one.mesh-delivery-watchman", password: "$2a$11$wwwwwwwwwwwwwwwwwwwwww", permissions: {
|
||||
publish: { allow: ["$JS.ACK.DEAD_LETTER_NOTICES.mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman_tools.>", "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.DIRECT.GET.PROV_APPLIED.>", "$JS.API.STREAM.INFO.DEAD_LETTERS", "$JS.API.STREAM.INFO.DEAD_LETTER_NOTICES", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.INFO.PROV_APPLIED", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS", "$JS.API.STREAM.MSG.GET.PROV_APPLIED", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.bus.connz", "mesh.events.dead.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~mesh-delivery-watchman", "mesh.seat.*.call.*.one~mesh-delivery-watchman", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.DEAD_LETTER_NOTICES.mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman.>", "$JS.ACK.EVENTS.one_mesh-delivery-watchman_tools.>", "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.CREATE.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.CONSUMER.DELETE.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES.mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.mesh-delivery-watchman", "$JS.API.DIRECT.GET.PROV_APPLIED.>", "$JS.API.STREAM.INFO.DEAD_LETTERS", "$JS.API.STREAM.INFO.DEAD_LETTER_NOTICES", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.INFO.PROV_APPLIED", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS", "$JS.API.STREAM.MSG.GET.PROV_APPLIED", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.bus.connz", "mesh.events.dead.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~mesh-delivery-watchman", "mesh.seat.*.call.*.one~mesh-delivery-watchman", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$SRV.INFO", "$SRV.INFO.mesh-delivery-watchman", "$SRV.INFO.mesh-delivery-watchman.>", "$SRV.PING", "$SRV.PING.mesh-delivery-watchman", "$SRV.PING.mesh-delivery-watchman.>", "$SRV.STATS", "$SRV.STATS.mesh-delivery-watchman", "$SRV.STATS.mesh-delivery-watchman.>", "_INBOX.one.mesh-delivery-watchman.>", "mesh.bus.connect", "mesh.bus.disconnect", "mesh.control.*.tools-alive", "mesh.events.dead.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.mesh-delivery-watchman.call.>", "mesh.mod.mesh-delivery-watchman.tool.>", "mesh.mod.nats.event.bus.connection-closed", "mesh.mod.nats.event.bus.slow-consumer", "mesh.mod.power.event.*", "mesh.seat.mesh-self-check.event.checked", "prov.applied.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
@@ -47,17 +47,17 @@ accounts {
|
||||
subscribe: { allow: ["_INBOX.one.nats.>"] }
|
||||
} }
|
||||
{ user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.EVENTS.one_telegram_tools.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.telegram", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.telegram", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~telegram", "mesh.seat.*.call.*.one~telegram", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.EVENTS.one_telegram_tools.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.CREATE.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.one.telegram", "$JS.API.CONSUMER.DELETE.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram_tools", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.one.telegram", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.one~telegram", "mesh.seat.*.call.*.one~telegram", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$JS.EVENT.ADVISORY.CONSUMER.CREATED.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.mod.telegram.call.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.ACK.EVENTS.two_audit_tools.>", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.audit", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.INFO.EVENTS.two_audit_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.audit", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~audit", "mesh.seat.*.call.*.two~audit", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.ACK.EVENTS.two_audit_tools.>", "$JS.API.CONSUMER.CREATE.EVENTS.two_audit_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.audit", "$JS.API.CONSUMER.DELETE.EVENTS.two_audit_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.INFO.EVENTS.two_audit_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.audit", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~audit", "mesh.seat.*.call.*.two~audit", "mesh.seat.*.tool.>"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.mod.audit.call.>", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: {
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.ACK.EVENTS.two_shop_tools.>", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.shop", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.INFO.EVENTS.two_shop_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.shop", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.mod.shop.event.order.placed", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~shop", "mesh.seat.*.call.*.two~shop", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.accept.send"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.ACK.EVENTS.two_shop_tools.>", "$JS.API.CONSUMER.CREATE.EVENTS.two_shop_tools", "$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.two.shop", "$JS.API.CONSUMER.DELETE.EVENTS.two_shop_tools", "$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.INFO.EVENTS.two_shop_tools", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop_tools", "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.two.shop", "$JS.API.STREAM.INFO.KV_mesh-grants_memberships", "$JS.API.STREAM.NAMES", "$JS.FC.KV_mesh-grants_memberships.>", "mesh.mod.shop.event.order.placed", "mesh.seat.*.accept.>", "mesh.seat.*.call.*.*.two~shop", "mesh.seat.*.call.*.two~shop", "mesh.seat.*.tool.>", "mesh.seat.telegram-sender.accept.send"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.seat.mesh-broker.call.apply.>", "mesh.seat.mesh-broker.call.create.>", "mesh.seat.mesh-broker.call.drop.>", "mesh.seat.mesh-broker.call.recreate.>", "mesh.seat.mesh-broker.tool.apply", "mesh.seat.mesh-broker.tool.create", "mesh.seat.mesh-broker.tool.drop", "mesh.seat.mesh-broker.tool.recreate"] }
|
||||
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.mod.shop.call.>", "mesh.mod.shop.tool.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -3,20 +3,75 @@ package buscompose
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
)
|
||||
|
||||
// A-397: beside each module's durable consumer the tool runner's twin, with the same filters under its own name, and the
|
||||
// events delivered again to one consumer named by the twin itself.
|
||||
func TestTheRunnersTwinBesideTheModulesConsumer(t *testing.T) {
|
||||
c := Consumer{Name: "ace_mod", Stream: "EVENTS", Filters: []string{"mesh.mod.x.event.y", AgainFilter("ace_mod")}}
|
||||
got := withToolsTwin(c)
|
||||
if len(got) != 2 || got[0].Name != "ace_mod" || got[1].Name != "ace_mod_tools" {
|
||||
t.Fatalf("consumers = %+v, want ace_mod and ace_mod_tools", got)
|
||||
// Design 47 H2, H-A-472 fifth to seventh addenda: the grants part composes no tool runner's twin; it writes the twin's whole
|
||||
// spec in the membership's heard entries, EVENTS like any stream, and grants the module's reader the create and delete of
|
||||
// its own twin by name.
|
||||
func TestTheRunnersTwinIsTheMembershipsSpec(t *testing.T) {
|
||||
c := Consumer{Name: "ace_mod", Stream: "EVENTS", Filters: []string{"mesh.mod.x.event.y", AgainFilter("ace_mod")}, AckWaitSeconds: 30,
|
||||
MaxDeliver: 5, FromNow: true}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
hasProgram bool
|
||||
want []string
|
||||
}{
|
||||
{"a module with a program of its own: the program's alone", true, []string{"ace_mod"}},
|
||||
{"a module the tool runner carries: none", false, nil},
|
||||
} {
|
||||
var got []string
|
||||
for _, tc := range twinsOf(c, row.hasProgram) {
|
||||
got = append(got, tc.Name)
|
||||
}
|
||||
if !slices.Equal(got, row.want) {
|
||||
t.Errorf("%s: consumers composed = %v, want %v", row.name, got, row.want)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got[1].Filters, []string{"mesh.mod.x.event.y", AgainFilter("ace_mod_tools")}) {
|
||||
t.Errorf("twin filters = %v", got[1].Filters)
|
||||
|
||||
d := Declared{Module: "mod", Consumes: []string{"x.happened"},
|
||||
Hears: []Heard{{Stream: "feed_one", Emitter: "feed", Filters: []string{"mesh.stream.feed.event.a"}},
|
||||
{Stream: "kept_one", Emitter: "keeper", Filters: []string{"mesh.stream.keeper.event.k"}, LastPerSubject: true}}}
|
||||
hears := HeardIssuedFor("ace", d)
|
||||
byStream := map[string]membership.HeardIssued{}
|
||||
for _, h := range hears {
|
||||
byStream[h.Stream] = h
|
||||
if !h.HasSpec() || h.Consumer != "ace_mod" || h.AckWaitSeconds != 30 || h.MaxDeliver != 5 {
|
||||
t.Errorf("heard entry %+v: want the twin's spec under the module's consumer ace_mod", h)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got[0].Filters, c.Filters) {
|
||||
t.Errorf("the module's filters changed: %v", got[0].Filters)
|
||||
for _, row := range []struct {
|
||||
stream, policy string
|
||||
filters []string
|
||||
}{
|
||||
{"EVENTS", membership.DeliverFromNow, []string{"mesh.mod.x.event.happened", AgainFilter("ace_mod_tools")}},
|
||||
{"feed_one", membership.DeliverFromNow, []string{"mesh.stream.feed.event.a"}},
|
||||
{"kept_one", membership.DeliverLastPerSubject, []string{"mesh.stream.keeper.event.k"}},
|
||||
} {
|
||||
h, ok := byStream[row.stream]
|
||||
if !ok {
|
||||
t.Errorf("no heard entry for %s in %+v", row.stream, hears)
|
||||
continue
|
||||
}
|
||||
if h.DeliverPolicy != row.policy || !slices.Equal(h.Filters, row.filters) {
|
||||
t.Errorf("%s: policy %q filters %v, want %q %v", row.stream, h.DeliverPolicy, h.Filters, row.policy, row.filters)
|
||||
}
|
||||
}
|
||||
|
||||
grants := readerGrants("EVENTS", "ace_mod")
|
||||
for _, row := range []struct {
|
||||
subject string
|
||||
want bool
|
||||
}{
|
||||
{"$JS.API.CONSUMER.CREATE.EVENTS.ace_mod_tools", true},
|
||||
{"$JS.API.CONSUMER.DELETE.EVENTS.ace_mod_tools", true},
|
||||
{"$JS.API.CONSUMER.INFO.EVENTS.ace_mod_tools", true},
|
||||
{"$JS.API.CONSUMER.CREATE.EVENTS.ace_mod", false},
|
||||
{"$JS.API.CONSUMER.DELETE.EVENTS.ace_mod", false},
|
||||
} {
|
||||
if slices.Contains(grants, row.subject) != row.want {
|
||||
t.Errorf("reader grants %s: %v, want %v (%v)", row.subject, !row.want, row.want, grants)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,7 +90,8 @@ type Declared struct {
|
||||
NoAccount bool
|
||||
// HasProgram says the module runs a program of its own that reads its consumer <node>_<module>: an account of its own
|
||||
// (an own broker secret) and a process resource with no `bus` of its own (design 47 H2, H-A-472 second addendum).
|
||||
// Without it, only the tool runner's twin <node>_<module>_tools is composed, so the pass makes no orphan.
|
||||
// Without it, nothing is composed on its consumer's name, so the pass makes no orphan; the tool runner's twin
|
||||
// <node>_<module>_tools is the runner's to make from the membership's spec (H-A-472 fifth to seventh addenda).
|
||||
HasProgram bool
|
||||
// SnapshotsTheBus says the module holds mesh-broker — it is the bus — and so is the one module
|
||||
// granted the snapshot API, to copy the bus's streams for the night's backup (novox/hq ADR 0235).
|
||||
|
||||
@@ -118,7 +118,8 @@ func durableTwin(name string) string {
|
||||
return name + "_tools"
|
||||
}
|
||||
|
||||
// orphaned says whether a held durable is an orphan (design 47 H2, H-A-472 (2)): no pulls waiting on it, while its twin
|
||||
// orphaned says whether a held durable is an orphan (design 47 H2, H-A-472 (2), eleventh addendum): never read at all, or
|
||||
// no pulls waiting on it, while its twin
|
||||
// on the same stream is current, with pulls waiting and a delivery within the bound (the setting twin-current-hours,
|
||||
// the third addendum). A module that moved into the tool runner left its program's durable so, and one that runs its own
|
||||
// program again its tool runner's.
|
||||
@@ -126,6 +127,11 @@ func orphaned(h heldConsumer, held map[string]heldConsumer, now time.Time, withi
|
||||
if h.WaitingPulls > 0 {
|
||||
return false
|
||||
}
|
||||
// A durable nobody ever read, no pull waiting and no delivery ever, is an orphan whatever its twin's state; one that was
|
||||
// read and lags is behind (H-A-472 eleventh addendum).
|
||||
if neverDelivered(h) {
|
||||
return true
|
||||
}
|
||||
t, ok := held[durableTwin(h.Name)]
|
||||
if !ok || t.WaitingPulls == 0 {
|
||||
return false
|
||||
@@ -134,6 +140,15 @@ func orphaned(h heldConsumer, held map[string]heldConsumer, now time.Time, withi
|
||||
return err == nil && now.Sub(last) < within
|
||||
}
|
||||
|
||||
// neverDelivered says the bus never delivered from the durable: no time of a last delivery, or the zero time.
|
||||
func neverDelivered(h heldConsumer) bool {
|
||||
if h.LastDelivered == "" {
|
||||
return true
|
||||
}
|
||||
last, err := time.Parse(time.RFC3339Nano, h.LastDelivered)
|
||||
return err == nil && last.IsZero()
|
||||
}
|
||||
|
||||
// heldConsumers reads the consumers the bus holds on stream, by name.
|
||||
func heldConsumers(c Caller, stream string) (map[string]heldConsumer, error) {
|
||||
raw, err := c.Call(CallStreams, map[string]any{"stream": stream})
|
||||
@@ -161,6 +176,25 @@ func heldConsumers(c Caller, stream string) (map[string]heldConsumer, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heardDurablesOf is, by stream, both twins of every heard entry of every assigned module's membership, the plain
|
||||
// `<consumer>` and the tool runner's `<consumer>_tools` (design 47 H2, H-A-472 tenth and eleventh addenda): each one held
|
||||
// and composed by no pass is judged by the behind check.
|
||||
func heardDurablesOf(recs buscompose.Records) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, node := range sortedKeys(recs.Assigned) {
|
||||
for _, d := range recs.Assigned[node] {
|
||||
for _, h := range buscompose.HeardIssuedFor(node, d) {
|
||||
for _, name := range []string{h.Consumer, h.Consumer + buscompose.ToolsSuffix} {
|
||||
if !slices.Contains(out[h.Stream], name) {
|
||||
out[h.Stream] = append(out[h.Stream], name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// filtersToAdd are the composed filters of a consumer the bus's one lacks, in the composed order; none when the consumer
|
||||
// filters the whole stream (no filters) or holds all.
|
||||
func filtersToAdd(c buscompose.Consumer, h heldConsumer) []string {
|
||||
@@ -200,12 +234,45 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
|
||||
for _, mc := range wanted {
|
||||
byStream[mc.Consumer.Stream] = append(byStream[mc.Consumer.Stream], mc.Consumer)
|
||||
}
|
||||
// The durables no pass composes, the tool runner's twins it makes itself from the membership's spec and the plain one of a
|
||||
// module whose program left, are still judged here, held on the streams each consuming module's heard entries name
|
||||
// (design 47 H2, H-A-472 tenth and eleventh addenda).
|
||||
twins := heardDurablesOf(recs)
|
||||
for stream := range twins {
|
||||
if _, composed := byStream[stream]; !composed {
|
||||
byStream[stream] = nil
|
||||
}
|
||||
}
|
||||
// judge is the behind check of one held consumer: an orphan is left out of it and named in a finding of its own
|
||||
// (H-A-472 (2)), so the alarm says a real backlog only.
|
||||
judge := func(key string, h heldConsumer, held map[string]heldConsumer) {
|
||||
if h.Pending <= BehindBound {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case !p.behindSeen[key]:
|
||||
case orphaned(h, held, p.now(), p.twinCurrentWithin()):
|
||||
orphans = append(orphans, key)
|
||||
default:
|
||||
st.Behind = append(st.Behind, fmt.Sprintf("%s: %d pending", key, h.Pending))
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
for _, stream := range sortedKeys(byStream) {
|
||||
held, err := heldConsumers(p.Call, stream)
|
||||
if err != nil {
|
||||
st.Unread = append(st.Unread, stream+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
composedHere := map[string]bool{}
|
||||
for _, c := range byStream[stream] {
|
||||
composedHere[c.Name] = true
|
||||
}
|
||||
for _, name := range twins[stream] {
|
||||
if h, ok := held[name]; ok && !composedHere[name] {
|
||||
judge(stream+"."+name, h, held)
|
||||
}
|
||||
}
|
||||
for _, c := range byStream[stream] {
|
||||
key := stream + "." + c.Name
|
||||
h, ok := held[c.Name]
|
||||
@@ -213,18 +280,7 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) {
|
||||
p.sayWorker(key + " is held already")
|
||||
}
|
||||
if ok {
|
||||
if h.Pending > BehindBound {
|
||||
// An orphan is left out of the behind check and named in a finding of its own (H-A-472 (2)), so the
|
||||
// alarm says a real backlog only.
|
||||
switch {
|
||||
case !p.behindSeen[key]:
|
||||
case orphaned(h, held, p.now(), p.twinCurrentWithin()):
|
||||
orphans = append(orphans, key)
|
||||
default:
|
||||
st.Behind = append(st.Behind, fmt.Sprintf("%s: %d pending", key, h.Pending))
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
judge(key, h, held)
|
||||
// A consumer whose module now consumes more events of the stream gains those filters by itself, only ever added:
|
||||
// a filter it holds that is no longer composed stays, and nothing else about it changes.
|
||||
if add := filtersToAdd(c, h); len(add) > 0 && st.On && p.Consumers != nil {
|
||||
@@ -453,8 +509,8 @@ func (p *Part) sayOrphaned(orphans []string) {
|
||||
needsYou(&c, "their removal, with the bus module's nats_consumer_remove", "These durables are more than the bound behind "+
|
||||
"their stream's head with no pulls waiting, while their module's twin on the same stream (<node>_<module> and "+
|
||||
"<node>_<module>_tools) has pulls waiting and delivered within "+p.twinCurrentWithin().String()+" (the setting "+
|
||||
"twin-current-hours): a module that moved into or out of the tool runner left them, and they are not said as "+
|
||||
"messages piling up: "+why)
|
||||
"twin-current-hours): a module that moved into or out of the tool runner left them, or a twin the runner made from "+
|
||||
"the membership and no longer reads, and they are not said as messages piling up: "+why)
|
||||
c.Resolved = "no orphaned durable is left"
|
||||
}
|
||||
if p.Say(c) == nil {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -332,3 +333,44 @@ func (h Holdings) CapabilitiesLost(last, now membership.Membership) []string {
|
||||
// CapabilityDifference opens what a membership's pass says when it was not published because it would drop a capability
|
||||
// its holder still asks for.
|
||||
const CapabilityDifference = ShadowDifference + "a capability its holder still asks for: "
|
||||
|
||||
// withdrawMemberships deletes the membership of a module no longer assigned on its node (design 47 H2, H-A-472 ninth
|
||||
// addendum): the tool runner there hears the delete and removes the module's `_tools` durables from the streams the entry
|
||||
// named. Only a whole pass withdraws: one that composes part of the mesh, or any of whose reads failed, keeps every key, as
|
||||
// the accounts are kept (accounts.go). A key of the state that is no membership (the bus module's bindings, BoundPrefix)
|
||||
// is never one.
|
||||
func (p *Part) withdrawMemberships(res *Result, in Inputs, composed map[string]Membership) {
|
||||
if in.keepsLoaded() != "" || len(in.ReadFailures()) > 0 {
|
||||
return
|
||||
}
|
||||
keys, err := p.Store.Keys(StateMemberships)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s cannot be listed, so no membership of a module gone is withdrawn: %v", StateMemberships, err))
|
||||
return
|
||||
}
|
||||
placed := Placed(in)
|
||||
for _, key := range keys {
|
||||
if _, still := composed[key]; still || strings.HasPrefix(key, BoundPrefix) || placed(key) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := p.Store.Get(StateMemberships, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
bare, err := membership.BareGrantsMembership(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var m membership.Membership
|
||||
if json.Unmarshal(bare, &m) != nil || buscompose.MembershipKey(m.Node, m.Module) != key {
|
||||
continue
|
||||
}
|
||||
if err := p.takeAway(StateMemberships, key); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s, of a module no longer assigned there, cannot be withdrawn: %v", StateMemberships, key, err))
|
||||
continue
|
||||
}
|
||||
if p.Logf != nil {
|
||||
p.Logf("the membership %s is withdrawn: %s is no longer assigned on %s", key, m.Module, m.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -572,6 +572,7 @@ func (p *Part) Compose() Result {
|
||||
res.Retired[key] = m.Retired
|
||||
}
|
||||
}
|
||||
p.withdrawMemberships(&res, in, ms)
|
||||
p.clearFull()
|
||||
held, heldErr := HeldStreams(p.Call)
|
||||
var ask Ask
|
||||
|
||||
Reference in New Issue
Block a user