Join fix/47-H1-indexer-take-in-again

This commit is contained in:
2026-11-21 00:26:46 +01:00
59 changed files with 1387 additions and 325 deletions
@@ -169,7 +169,7 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
changed := false
var kept []tierMover
defer func() {
if err := e.startTiers(p.Pipeline, kept); err != nil {
if err := e.startTiers(p.Pipeline, p.Source, kept); err != nil {
e.logf("the tier progress of %s is not written: %v", p.Pipeline, err)
}
}()
@@ -42,11 +42,14 @@ type tierMover struct {
Tier int64 `json:"tier"`
}
// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, and the movers.
// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, the movers, and the source
// revision its pipeline-ended names, said in pipeline-wanted beside the key's revision (design 47 H-A-214, H-A-216); zero in a
// value kept before the field, read as the key's number.
type tierProgress struct {
Tier int64 `json:"tier"`
Since time.Time `json:"since"`
Movers []tierMover `json:"movers"`
Source int64 `json:"source,omitempty"`
}
// tierKey is the key of pipeline's tier progress; pipeline is the runner's key `<branch>.<source>`, its branch escaped.
@@ -60,7 +63,7 @@ func isTierKey(key string) bool {
// startTiers writes the tier progress of a passed pipeline whose movers' versions are now kept, its first tier the lowest of
// theirs, since now. A pipeline heard again writes nothing.
func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error {
func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMover) error {
if len(movers) == 0 {
return nil
}
@@ -68,7 +71,7 @@ func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error {
for _, m := range movers {
first = min(first, m.Tier)
}
value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers})
value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers, Source: source})
if err != nil {
return err
}
@@ -131,7 +134,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st
continue
}
pipeline := strings.TrimPrefix(key, tierPrefix)
branchEsc, source, _ := strings.Cut(pipeline, ".")
branchEsc, number, _ := strings.Cut(pipeline, ".")
branch, err := keys.UnescapeName(branchEsc)
if err != nil {
continue
@@ -144,7 +147,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st
notRunning := p.notRunning(n, placedOn, pipeline, branch, t, t.Tier)
late := !now.Before(t.Since.Add(bound))
if late {
p.tierLate(pipeline, branch, source, t, notRunning)
p.tierLate(pipeline, branch, number, t, notRunning)
break
}
if len(notRunning) > 0 {
@@ -165,10 +168,10 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st
p.logf("the tier progress %s is not deleted: %v", key, err)
break
}
p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeRunning, nil)
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeRunning, nil)
break
}
value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers})
value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers, Source: t.Source})
if err == nil {
err = p.ends.st.swap(key, value, rev)
}
@@ -217,7 +220,7 @@ func (p *passer) notRunning(n assign.Nodes, placedOn func(module, branch string)
// tierLate ends a pipeline's delivery at a tier past tier-bound: the finding tier-late is raised and pipeline-wanted says tier-late,
// once each; the finding clears once the late movers run or a later pipeline moved them. The progress stays, so no later tier is
// written, until no later tier's mover is kept at this pipeline any more.
func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRunning []meshwants.NotRunning) {
func (p *passer) tierLate(pipeline, branch, number string, t tierProgress, notRunning []meshwants.NotRunning) {
o := tierLateFinding(pipeline, t.Tier, notRunning)
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
if len(notRunning) > 0 {
@@ -231,7 +234,7 @@ func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRu
p.said[key+".raised"] = true
delete(p.said, key+".cleared")
p.logf("pipeline %s: tier %d is late (tier-bound %s): %s; no later tier is written", pipeline, t.Tier, p.tierBoundOf(), o.Summary)
p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeTierLate, notRunning)
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeTierLate, notRunning)
}
} else if !p.said[key+".cleared"] {
if p.findings.Delete != nil {
@@ -271,11 +274,16 @@ func tierLateFinding(pipeline string, tier int64, notRunning []meshwants.NotRunn
Source: "assigner." + kindTierLate}
}
// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome).
func (p *passer) sayPipelineWanted(pipeline, branch, source, outcome string, notRunning []meshwants.NotRunning) {
var src int64
_, _ = fmt.Sscan(source, &src)
body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: src, Outcome: outcome, NotRunning: notRunning}
// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome): its
// revision the number of the pipeline's key `<branch>.<revision>`, its source the one the pipeline-ended named (design 47
// H-A-214, H-A-216), the key's number for a progress kept before it held one.
func (p *passer) sayPipelineWanted(pipeline, branch, number string, source int64, outcome string, notRunning []meshwants.NotRunning) {
var revision int64
_, _ = fmt.Sscan(number, &revision)
if source <= 0 {
source = revision
}
body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: source, Revision: revision, Outcome: outcome, NotRunning: notRunning}
event := meshwants.EventPipelineWanted + "." + pipeline
if err := p.wantsSay(event, body, "pipeline-wanted."+pipeline+"."+outcome); err != nil {
p.logf("pipeline-wanted of %s is not said: %v", pipeline, err)
@@ -62,8 +62,11 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
p.w.mu.Unlock()
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now }
if _, err := p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))); err != nil {
// The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216).
e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))
e.Source = 4
if _, err := p.ends.take(e); err != nil {
t.Fatal(err)
}
wanted := func(node, module string) bool {
@@ -88,7 +91,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web})
_ = p.w.hearRunning("running.g14.web", raw)
p.wants(context.Background())
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || len(findings) != 0 {
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 {
t.Errorf("every tier runs: said %+v, findings %v", said, findings)
}
if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok {
@@ -108,8 +111,14 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") {
t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings)
}
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" {
t.Errorf("pipeline-wanted tier-late is not said once: %+v", said)
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" ||
said[0].Revision != 5 || said[0].Source != 4 {
t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said)
}
// A tier progress kept before it held its source says the key's number as both (H-A-216).
p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 {
t.Errorf("a progress with no source: said %+v, want revision and source 5", said)
}
if wanted("g14", "web") || wanted("ace", "web") {
t.Error("a consumer's want is written after its provider's tier was late")
@@ -715,6 +715,10 @@ type CatalogRevision struct {
Base int64 `json:"base,omitempty"`
// the watched branch the revision is of
Branch string `json:"branch,omitempty"`
// what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of <module> <version>, a take-in again's take-in again of <repository>; absent for a candidate's
Call string `json:"call,omitempty"`
// the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's
CallKind string `json:"call-kind,omitempty"`
// the seat definitions it was judged with, each by its digest, read whole through seat-definitions
Definitions []CatalogRevisionDefinitionsItem `json:"definitions,omitempty"`
// one per module, by name
@@ -725,8 +729,12 @@ type CatalogRevision struct {
Heads string `json:"heads,omitempty"`
// trunk or candidate
Kind string `json:"kind"`
// for call-kind release, the module released (H-A-215)
Module string `json:"module,omitempty"`
// the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover
Movers []TakeInEntry `json:"movers,omitempty"`
// for call-kind take-in-again, the repository taken in again (H-A-215)
Repository string `json:"repository,omitempty"`
// the catalog graph revision's number, counted by its holder
Revision int64 `json:"revision"`
// the source revision it was taken in from; for a candidate, the source revision its heads are merged onto
@@ -735,8 +743,18 @@ type CatalogRevision struct {
Trunk []Head `json:"trunk,omitempty"`
// the layout version
V int64 `json:"v,omitempty"`
// for call-kind release, the version released (H-A-215)
Version string `json:"version,omitempty"`
}
// The values of CatalogRevision's call-kind.
const (
CatalogRevisionCallKindTakeIn = "take-in"
CatalogRevisionCallKindRelease = "release"
CatalogRevisionCallKindTakeInAgain = "take-in-again"
CatalogRevisionCallKindRecompute = "recompute"
)
// Edge is one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses).
type Edge struct {
// declared in the artifact entry's needs of A's manifest, where nothing else says it
@@ -1301,6 +1319,10 @@ type CatalogRevisionEvent struct {
At *time.Time `json:"at,omitempty"`
// the watched branch the revision is of, escaped as a key name is
Branch string `json:"branch,omitempty"`
// what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's
Call string `json:"call,omitempty"`
// the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's
CallKind string `json:"call-kind,omitempty"`
// the modules whose entries differ from the revision before it of the same kind
Changed []string `json:"changed,omitempty"`
// how many entries it lists
@@ -1308,12 +1330,26 @@ type CatalogRevisionEvent struct {
Group string `json:"group,omitempty"`
Heads string `json:"heads,omitempty"`
// trunk or candidate: a candidate's moves no running part
Kind string `json:"kind"`
Revision int64 `json:"revision"`
Source int64 `json:"source"`
V int64 `json:"v"`
Kind string `json:"kind"`
// for call-kind release, the module released (H-A-215)
Module string `json:"module,omitempty"`
// for call-kind take-in-again, the repository taken in again (H-A-215)
Repository string `json:"repository,omitempty"`
Revision int64 `json:"revision"`
Source int64 `json:"source"`
V int64 `json:"v"`
// for call-kind release, the version released (H-A-215)
Version string `json:"version,omitempty"`
}
// The values of CatalogRevisionEvent's call-kind.
const (
CatalogRevisionEventCallKindTakeIn = "take-in"
CatalogRevisionEventCallKindRelease = "release"
CatalogRevisionEventCallKindTakeInAgain = "take-in-again"
CatalogRevisionEventCallKindRecompute = "recompute"
)
// FillingEvent is the body of the event filling.
type FillingEvent = Fill
@@ -17,7 +17,7 @@ func TestFixtures(t *testing.T) {
decode func([]byte) (any, error)
subjectIs *regexp.Regexp
}{
{name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}",
{name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}",
decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")},
{name: "a candidate's revision kept", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")},
@@ -59,7 +59,7 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v ModulesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "the module verb (captured: xclip)", subject: "", given: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", wire: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}",
decode: func(b []byte) (any, error) { var v ModuleAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}",
{name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}",
decode: func(b []byte) (any, error) {
var v CatalogRevisionsByBranchRevisionValue
err := json.Unmarshal(b, &v)
@@ -65,9 +65,9 @@ const (
// StreamPipelineStarted is the stream pipeline-started is kept in.
const StreamPipelineStarted = "SEAT_mesh-pipeline-runner_pipeline-started"
// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started.<branch>.<source>.
func SubjectPipelineStarted(branch, source string) string {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source
// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started.<branch>.<revision>.
func SubjectPipelineStarted(branch, revision string) string {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision
}
// SubjectPipelineStartedAll hears every pipeline-started.
@@ -76,9 +76,9 @@ const SubjectPipelineStartedAll = "mesh.seat.mesh-pipeline-runner.event.pipeline
// StreamPipelineEnded is the stream pipeline-ended is kept in.
const StreamPipelineEnded = "SEAT_mesh-pipeline-runner_pipeline-ended"
// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended.<branch>.<source>.
func SubjectPipelineEnded(branch, source string) string {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source
// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended.<branch>.<revision>.
func SubjectPipelineEnded(branch, revision string) string {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision
}
// SubjectPipelineEndedAll hears every pipeline-ended.
@@ -120,9 +120,9 @@ const SubjectStepEndedAll = "mesh.seat.mesh-pipeline-runner.event.step-ended.*.*
// StreamWaitHeld is the stream wait-held is kept in.
const StreamWaitHeld = "SEAT_mesh-pipeline-runner_wait-held"
// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held.<branch>.<source>.
func SubjectWaitHeld(branch, source string) string {
return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source
// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held.<branch>.<revision>.
func SubjectWaitHeld(branch, revision string) string {
return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision
}
// SubjectWaitHeldAll hears every wait-held.
@@ -131,9 +131,9 @@ const SubjectWaitHeldAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"
// StreamWaitEnded is the stream wait-ended is kept in.
const StreamWaitEnded = "SEAT_mesh-pipeline-runner_wait-ended"
// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended.<branch>.<source>.
func SubjectWaitEnded(branch, source string) string {
return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source
// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended.<branch>.<revision>.
func SubjectWaitEnded(branch, revision string) string {
return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision
}
// SubjectWaitEndedAll hears every wait-ended.
@@ -156,11 +156,11 @@ const (
StateStepOutputs = "step-outputs"
)
// PipelinesByBranchSourceKey is the key of one value of the state pipelines under the form "<branch>.<source>".
func PipelinesByBranchSourceKey(branch, source string) string { return branch + "." + source }
// PipelinesByBranchRevisionKey is the key of one value of the state pipelines under the form "<branch>.<revision>".
func PipelinesByBranchRevisionKey(branch, revision string) string { return branch + "." + revision }
// ParsePipelinesByBranchSourceKey reads the parts of a key of the form "<branch>.<source>", in the order of the form; the last result is false for a key of another form.
func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) {
// ParsePipelinesByBranchRevisionKey reads the parts of a key of the form "<branch>.<revision>", in the order of the form; the last result is false for a key of another form.
func ParsePipelinesByBranchRevisionKey(key string) (string, string, bool) {
rest := key
v0, after0, found0 := strings.Cut(rest, ".")
if !found0 || v0 == "" {
@@ -178,9 +178,9 @@ func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) {
return v0, v2, true
}
// DecodePipelinesByBranchSourceValue reads one value of the state pipelines under the form "<branch>.<source>", refusing a field PipelinesByBranchSourceValue does not know and any text after the value.
func DecodePipelinesByBranchSourceValue(raw []byte) (PipelinesByBranchSourceValue, error) {
var v PipelinesByBranchSourceValue
// DecodePipelinesByBranchRevisionValue reads one value of the state pipelines under the form "<branch>.<revision>", refusing a field PipelinesByBranchRevisionValue does not know and any text after the value.
func DecodePipelinesByBranchRevisionValue(raw []byte) (PipelinesByBranchRevisionValue, error) {
var v PipelinesByBranchRevisionValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
@@ -670,7 +670,7 @@ type Pipeline struct {
// the modules the pipeline moves, its take-in's entries (design 47 H §746)
Movers []Mover `json:"movers"`
Outcome string `json:"outcome"`
// the pipeline's key, <branch>.<source>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)
// the pipeline's key, <branch>.<revision>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)
Pipeline string `json:"pipeline"`
// for a pipeline a package's publish started, that publish (A-202)
Published *Published `json:"published,omitempty"`
@@ -720,7 +720,7 @@ type PipelineEnded struct {
// per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2)
Movers []MoverEnded `json:"movers"`
Outcome string `json:"outcome"`
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
// the catalog graph revision of the branch the pipeline runs for
Revision int64 `json:"revision,omitempty"`
@@ -753,7 +753,7 @@ type PipelineStarted struct {
Call string `json:"call,omitempty"`
// the pipeline's movers
Movers []Mover `json:"movers"`
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
// for a pipeline a package's publish started, that publish (A-202)
Published *Published `json:"published,omitempty"`
@@ -814,7 +814,7 @@ type StepEvent struct {
// the module's build-input fingerprint at the commit
Fingerprint string `json:"fingerprint,omitempty"`
Module string `json:"module"`
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
Result StepResult `json:"result"`
// the catalog graph revision of the branch the pipeline runs for
@@ -966,7 +966,7 @@ type ToolchainUsed struct {
Version string `json:"version"`
}
// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<source> (design 47 H-A-72).
// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<revision> (design 47 H-A-72, H-A-221).
type Wait struct {
Artifact string `json:"artifact,omitempty"`
// the step's own ask, where it has one (design 47 G §731)
@@ -983,7 +983,7 @@ type Wait struct {
Module string `json:"module,omitempty"`
// what the step waits on (design 47 *Health and going back*)
On string `json:"on"`
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
// the catalog graph revision of the branch the pipeline runs for
Revision int64 `json:"revision,omitempty"`
@@ -1024,7 +1024,7 @@ const (
// Waiting is a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72).
type Waiting struct {
// the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
Pipeline string `json:"pipeline"`
}
@@ -1355,8 +1355,8 @@ type FullEvent = Fill
// RoomEvent is the body of the event room.
type RoomEvent = Fill
// PipelinesByBranchSourceValue is one value of the state pipelines under the key <branch>.<source>.
type PipelinesByBranchSourceValue = Pipeline
// PipelinesByBranchRevisionValue is one value of the state pipelines under the key <branch>.<revision>.
type PipelinesByBranchRevisionValue = Pipeline
// PipelinesByBranchSourcePublishedPackageVersionValue is one value of the state pipelines under the key <branch>.<source>.published.<package>.<version>.
type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline
@@ -45,7 +45,7 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a take-in's pipeline kept, its steps naming their mover", subject: "", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}",
decode: func(b []byte) (any, error) {
var v PipelinesByBranchSourceValue
var v PipelinesByBranchRevisionValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
@@ -73,13 +73,13 @@ func TestFixtures(t *testing.T) {
}, subjectIs: nil},
{name: "a pipeline waiting its turn, its seq given", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}",
decode: func(b []byte) (any, error) {
var v PipelinesByBranchSourceValue
var v PipelinesByBranchRevisionValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
{name: "a waiting pipeline names the movers a newer take-in took out of it", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}",
decode: func(b []byte) (any, error) {
var v PipelinesByBranchSourceValue
var v PipelinesByBranchRevisionValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
@@ -83,9 +83,9 @@ const SubjectWantedAll = "mesh.seat.mesh-wants.event.wanted.*.*"
// StreamPipelineWanted is the stream pipeline-wanted is kept in.
const StreamPipelineWanted = "SEAT_mesh-wants_pipeline-wanted"
// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted.<branch>.<source>.
func SubjectPipelineWanted(branch, source string) string {
return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source
// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted.<branch>.<revision>.
func SubjectPipelineWanted(branch, revision string) string {
return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision
}
// SubjectPipelineWantedAll hears every pipeline-wanted.
@@ -1001,10 +1001,12 @@ type PipelineWanted struct {
// for tier-late, the late tier's movers not running and the machines they do not run on
NotRunning []NotRunning `json:"not-running,omitempty"`
Outcome string `json:"outcome"`
// the runner's pipeline key, <branch>.<source>, a branch escaped as a key's name is (design 47 H §746)
// the runner's pipeline key, <branch>.<revision>, a branch escaped as a key's name is (design 47 H §746, H-A-213 (4))
Pipeline string `json:"pipeline"`
Source int64 `json:"source"`
V int64 `json:"v"`
// the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event
Revision int64 `json:"revision,omitempty"`
Source int64 `json:"source"`
V int64 `json:"v"`
}
// The values of PipelineWanted's outcome.
@@ -29,7 +29,7 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a want removed, its node left or its module unplaced", subject: "mesh.seat.mesh-wants.event.wanted.n1.xclip", given: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}",
decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}",
{name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a pipeline's tier late, naming the movers and machines not running", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}", wire: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}",
decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")},
@@ -8,6 +8,10 @@
"revision": 12,
"kind": "trunk",
"source": 3589,
"call": "release of mesh-sdk 0.1.27",
"call-kind": "release",
"module": "mesh-sdk",
"version": "0.1.27",
"entries": 2,
"changed": [
"xclip"
@@ -209,7 +209,11 @@
"repository": "novox/mesh-catalog"
}
],
"base": 9
"base": 9,
"call": "release of mesh-sdk 0.1.27",
"call-kind": "release",
"module": "mesh-sdk",
"version": "0.1.27"
}
},
{
@@ -657,6 +657,32 @@
"source": {
"type": "integer"
},
"call": {
"type": "string",
"description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's"
},
"call-kind": {
"type": "string",
"enum": [
"take-in",
"release",
"take-in-again",
"recompute"
],
"description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's"
},
"module": {
"type": "string",
"description": "for call-kind release, the module released (H-A-215)"
},
"version": {
"type": "string",
"description": "for call-kind release, the version released (H-A-215)"
},
"repository": {
"type": "string",
"description": "for call-kind take-in-again, the repository taken in again (H-A-215)"
},
"group": {
"type": "string"
},
@@ -1871,6 +1897,32 @@
"type": "integer",
"description": "the source revision it was taken in from; for a candidate, the source revision its heads are merged onto"
},
"call": {
"type": "string",
"description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of <module> <version>, a take-in again's take-in again of <repository>; absent for a candidate's"
},
"call-kind": {
"type": "string",
"enum": [
"take-in",
"release",
"take-in-again",
"recompute"
],
"description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's"
},
"module": {
"type": "string",
"description": "for call-kind release, the module released (H-A-215)"
},
"version": {
"type": "string",
"description": "for call-kind release, the version released (H-A-215)"
},
"repository": {
"type": "string",
"description": "for call-kind take-in-again, the repository taken in again (H-A-215)"
},
"group": {
"type": "string",
"description": "a candidate's group"
@@ -1,7 +1,7 @@
[
{
"name": "a take-in's pipeline kept, its steps naming their mover",
"of": "state:pipelines/<branch>.<source>",
"of": "state:pipelines/<branch>.<revision>",
"given": {
"pipeline": "main.3589",
"branch": "main",
@@ -200,7 +200,7 @@
},
{
"name": "a pipeline waiting its turn, its seq given",
"of": "state:pipelines/<branch>.<source>",
"of": "state:pipelines/<branch>.<revision>",
"given": {
"pipeline": "main.3590",
"branch": "main",
@@ -224,7 +224,7 @@
},
{
"name": "a waiting pipeline names the movers a newer take-in took out of it",
"of": "state:pipelines/<branch>.<source>",
"of": "state:pipelines/<branch>.<revision>",
"given": {
"pipeline": "main.3590",
"branch": "main",
@@ -672,7 +672,7 @@
"events": [
{
"name": "pipeline-started",
"subject": "pipeline-started.<branch>.<source>",
"subject": "pipeline-started.<branch>.<revision>",
"body": {
"$ref": "#/defs/PipelineStarted"
},
@@ -683,7 +683,7 @@
},
{
"name": "pipeline-ended",
"subject": "pipeline-ended.<branch>.<source>",
"subject": "pipeline-ended.<branch>.<revision>",
"body": {
"$ref": "#/defs/PipelineEnded"
},
@@ -727,7 +727,7 @@
},
{
"name": "wait-held",
"subject": "wait-held.<branch>.<source>",
"subject": "wait-held.<branch>.<revision>",
"body": {
"$ref": "#/defs/Wait"
},
@@ -738,7 +738,7 @@
},
{
"name": "wait-ended",
"subject": "wait-ended.<branch>.<source>",
"subject": "wait-ended.<branch>.<revision>",
"body": {
"$ref": "#/defs/Wait"
},
@@ -778,11 +778,11 @@
"description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)",
"key-forms": [
{
"key": "<branch>.<source>",
"key": "<branch>.<revision>",
"value": {
"$ref": "#/defs/Pipeline"
},
"description": "a take-in's pipeline"
"description": "a pipeline of one catalog revision, keyed by the revision's number (design 47 H-A-213 (4), H-A-219); one kept from before under its source has the same form"
},
{
"key": "<branch>.<source>.published.<package>.<version>",
@@ -1199,7 +1199,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<source>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)"
"description": "the pipeline's key, <branch>.<revision>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)"
},
"branch": {
"type": "string",
@@ -1313,7 +1313,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
},
"branch": {
"type": "string",
@@ -1388,7 +1388,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
},
"branch": {
"type": "string",
@@ -1495,7 +1495,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
},
"module": {
"type": "string"
@@ -1550,7 +1550,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
},
"module": {
"type": "string",
@@ -1635,7 +1635,7 @@
"description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)"
}
},
"description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<source> (design 47 H-A-72)",
"description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<revision> (design 47 H-A-72, H-A-221)",
"required": [
"v",
"pipeline",
@@ -1978,7 +1978,7 @@
"properties": {
"pipeline": {
"type": "string",
"description": "the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
"description": "the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)"
}
},
"description": "a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72)",
@@ -65,12 +65,13 @@
{
"name": "a pipeline's wants all run",
"of": "event:pipeline-wanted",
"subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589",
"subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590",
"given": {
"v": 1,
"pipeline": "main.3589",
"pipeline": "main.3590",
"branch": "main",
"source": 3589,
"revision": 3590,
"outcome": "running"
}
},
@@ -434,7 +434,7 @@
},
{
"name": "pipeline-wanted",
"subject": "pipeline-wanted.<branch>.<source>",
"subject": "pipeline-wanted.<branch>.<revision>",
"keep": {
"max-age-days": 30,
"max-mib": 64
@@ -1725,7 +1725,7 @@
},
"pipeline": {
"type": "string",
"description": "the runner's pipeline key, <branch>.<source>, a branch escaped as a key's name is (design 47 H §746)"
"description": "the runner's pipeline key, <branch>.<revision>, a branch escaped as a key's name is (design 47 H §746, H-A-213 (4))"
},
"branch": {
"type": "string"
@@ -1733,6 +1733,10 @@
"source": {
"type": "integer"
},
"revision": {
"type": "integer",
"description": "the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event"
},
"outcome": {
"type": "string",
"enum": [
@@ -277,6 +277,10 @@ export interface CatalogRevision {
"base"?: number;
/** the watched branch the revision is of */
"branch"?: string;
/** what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of <module> <version>, a take-in again's take-in again of <repository>; absent for a candidate's */
"call"?: string;
/** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's */
"call-kind"?: "take-in" | "release" | "take-in-again" | "recompute";
/** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */
"definitions"?: CatalogRevisionDefinitionsItem[];
/** one per module, by name */
@@ -287,8 +291,12 @@ export interface CatalogRevision {
"heads"?: string;
/** trunk or candidate */
"kind": string;
/** for call-kind release, the module released (H-A-215) */
"module"?: string;
/** the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover */
"movers"?: TakeInEntry[];
/** for call-kind take-in-again, the repository taken in again (H-A-215) */
"repository"?: string;
/** the catalog graph revision's number, counted by its holder */
"revision": number;
/** the source revision it was taken in from; for a candidate, the source revision its heads are merged onto */
@@ -297,6 +305,8 @@ export interface CatalogRevision {
"trunk"?: Head[];
/** the layout version */
"v"?: number;
/** for call-kind release, the version released (H-A-215) */
"version"?: string;
}
/** one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses) */
@@ -796,6 +806,10 @@ export interface CatalogRevisionEvent {
"at"?: string;
/** the watched branch the revision is of, escaped as a key name is */
"branch"?: string;
/** what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's */
"call"?: string;
/** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's */
"call-kind"?: "take-in" | "release" | "take-in-again" | "recompute";
/** the modules whose entries differ from the revision before it of the same kind */
"changed"?: string[];
/** how many entries it lists */
@@ -804,9 +818,15 @@ export interface CatalogRevisionEvent {
"heads"?: string;
/** trunk or candidate: a candidate's moves no running part */
"kind": string;
/** for call-kind release, the module released (H-A-215) */
"module"?: string;
/** for call-kind take-in-again, the repository taken in again (H-A-215) */
"repository"?: string;
"revision": number;
"source": number;
"v": number;
/** for call-kind release, the version released (H-A-215) */
"version"?: string;
}
/** is the body of the event filling. */
@@ -40,13 +40,13 @@ export declare const family: { readonly name: "mesh-pipeline-runner.events"; rea
export declare const pipelineStartedStream: "SEAT_mesh-pipeline-runner_pipeline-started";
export declare function pipelineStartedSubject(branch: string, source: string): string;
export declare function pipelineStartedSubject(branch: string, revision: string): string;
export declare const pipelineStartedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*";
export declare const pipelineEndedStream: "SEAT_mesh-pipeline-runner_pipeline-ended";
export declare function pipelineEndedSubject(branch: string, source: string): string;
export declare function pipelineEndedSubject(branch: string, revision: string): string;
export declare const pipelineEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*";
@@ -70,13 +70,13 @@ export declare const stepEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.
export declare const waitHeldStream: "SEAT_mesh-pipeline-runner_wait-held";
export declare function waitHeldSubject(branch: string, source: string): string;
export declare function waitHeldSubject(branch: string, revision: string): string;
export declare const waitHeldSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*";
export declare const waitEndedStream: "SEAT_mesh-pipeline-runner_wait-ended";
export declare function waitEndedSubject(branch: string, source: string): string;
export declare function waitEndedSubject(branch: string, revision: string): string;
export declare const waitEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*";
@@ -94,11 +94,11 @@ export declare const states: {
readonly stepOutputs: "step-outputs";
};
/** The key of one value of the state pipelines under the form "<branch>.<source>". */
export declare function pipelinesByBranchSourceKey(branch: string, source: string): string;
/** The key of one value of the state pipelines under the form "<branch>.<revision>". */
export declare function pipelinesByBranchRevisionKey(branch: string, revision: string): string;
/** The parts of a key of the form "<branch>.<source>", or undefined for a key of another form. */
export declare function parsePipelinesByBranchSourceKey(_key: string): { branch: string; source: string } | undefined;
/** The parts of a key of the form "<branch>.<revision>", or undefined for a key of another form. */
export declare function parsePipelinesByBranchRevisionKey(_key: string): { branch: string; revision: string } | undefined;
/** The key of one value of the state pipelines under the form "<branch>.<source>.published.<package>.<version>". */
export declare function pipelinesByBranchSourcePublishedPackageVersionKey(branch: string, source: string, packagePart: string, version: string): string;
@@ -261,7 +261,7 @@ export interface Pipeline {
/** the modules the pipeline moves, its take-in's entries (design 47 H §746) */
"movers": Mover[];
"outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
/** the pipeline's key, <branch>.<source>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */
/** the pipeline's key, <branch>.<revision>, <branch>.<source>.published.<package>.<version> for a pipeline a publish started, or <branch>.run.<run> for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */
"pipeline": string;
/** for a pipeline a package's publish started, that publish (A-202) */
"published"?: Published;
@@ -300,7 +300,7 @@ export interface PipelineEnded {
/** per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) */
"movers": MoverEnded[];
"outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
/** the catalog graph revision of the branch the pipeline runs for */
"revision"?: number;
@@ -322,7 +322,7 @@ export interface PipelineStarted {
"call"?: string;
/** the pipeline's movers */
"movers": Mover[];
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
/** for a pipeline a package's publish started, that publish (A-202) */
"published"?: Published;
@@ -383,7 +383,7 @@ export interface StepEvent {
/** the module's build-input fingerprint at the commit */
"fingerprint"?: string;
"module": string;
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
"result": StepResult;
/** the catalog graph revision of the branch the pipeline runs for */
@@ -490,7 +490,7 @@ export interface ToolchainUsed {
"version": string;
}
/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<source> (design 47 H-A-72) */
/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on <branch>.<revision> (design 47 H-A-72, H-A-221) */
export interface Wait {
"artifact"?: string;
/** the step's own ask, where it has one (design 47 G §731) */
@@ -507,7 +507,7 @@ export interface Wait {
"module"?: string;
/** what the step waits on (design 47 *Health and going back*) */
"on": "person" | "build node" | "revision view" | "pipelines-at-once";
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
/** the catalog graph revision of the branch the pipeline runs for */
"revision"?: number;
@@ -527,7 +527,7 @@ export interface Wait {
/** a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72) */
export interface Waiting {
/** the pipeline's key, <branch>.<source>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
"pipeline": string;
}
@@ -800,8 +800,8 @@ export type FullEvent = Fill;
/** is the body of the event room. */
export type RoomEvent = Fill;
/** is one value of the state pipelines under the key <branch>.<source>. */
export type PipelinesByBranchSourceValue = Pipeline;
/** is one value of the state pipelines under the key <branch>.<revision>. */
export type PipelinesByBranchRevisionValue = Pipeline;
/** is one value of the state pipelines under the key <branch>.<source>.published.<package>.<version>. */
export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline;
@@ -40,16 +40,16 @@ export const family = { name: "mesh-pipeline-runner.events", maxAgeDays: 7 };
export const pipelineStartedStream = "SEAT_mesh-pipeline-runner_pipeline-started";
export function pipelineStartedSubject(branch, source) {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source;
export function pipelineStartedSubject(branch, revision) {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision;
}
export const pipelineStartedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*";
export const pipelineEndedStream = "SEAT_mesh-pipeline-runner_pipeline-ended";
export function pipelineEndedSubject(branch, source) {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source;
export function pipelineEndedSubject(branch, revision) {
return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision;
}
export const pipelineEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*";
@@ -80,16 +80,16 @@ export const stepEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.step-en
export const waitHeldStream = "SEAT_mesh-pipeline-runner_wait-held";
export function waitHeldSubject(branch, source) {
return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source;
export function waitHeldSubject(branch, revision) {
return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision;
}
export const waitHeldSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*";
export const waitEndedStream = "SEAT_mesh-pipeline-runner_wait-ended";
export function waitEndedSubject(branch, source) {
return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source;
export function waitEndedSubject(branch, revision) {
return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision;
}
export const waitEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*";
@@ -108,23 +108,23 @@ export const states = {
stepOutputs: "step-outputs",
};
/** The key of one value of the state pipelines under the form "<branch>.<source>". */
export function pipelinesByBranchSourceKey(branch, source) {
return branch + "." + source;
/** The key of one value of the state pipelines under the form "<branch>.<revision>". */
export function pipelinesByBranchRevisionKey(branch, revision) {
return branch + "." + revision;
}
/** The parts of a key of the form "<branch>.<source>", or undefined for a key of another form. */
export function parsePipelinesByBranchSourceKey(_key) {
/** The parts of a key of the form "<branch>.<revision>", or undefined for a key of another form. */
export function parsePipelinesByBranchRevisionKey(_key) {
let _rest = _key;
const _at0 = _rest.indexOf(".");
if (_at0 <= 0) return undefined;
const branch = _rest.slice(0, _at0);
_rest = _rest.slice(_at0 + 1);
if (_rest === "") return undefined;
const source = _rest;
const revision = _rest;
_rest = "";
if (_rest !== "") return undefined;
return { branch, source };
return { branch, revision };
}
/** The key of one value of the state pipelines under the form "<branch>.<source>.published.<package>.<version>". */
@@ -50,7 +50,7 @@ export declare const wantedSubjectAll: "mesh.seat.mesh-wants.event.wanted.*.*";
export declare const pipelineWantedStream: "SEAT_mesh-wants_pipeline-wanted";
export declare function pipelineWantedSubject(branch: string, source: string): string;
export declare function pipelineWantedSubject(branch: string, revision: string): string;
export declare const pipelineWantedSubjectAll: "mesh.seat.mesh-wants.event.pipeline-wanted.*.*";
@@ -412,8 +412,10 @@ export interface PipelineWanted {
/** for tier-late, the late tier's movers not running and the machines they do not run on */
"not-running"?: NotRunning[];
"outcome": "running" | "tier-late";
/** the runner's pipeline key, <branch>.<source>, a branch escaped as a key's name is (design 47 H §746) */
/** the runner's pipeline key, <branch>.<revision>, a branch escaped as a key's name is (design 47 H §746, H-A-213 (4)) */
"pipeline": string;
/** the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event */
"revision"?: number;
"source": number;
"v": number;
}
@@ -52,8 +52,8 @@ export const wantedSubjectAll = "mesh.seat.mesh-wants.event.wanted.*.*";
export const pipelineWantedStream = "SEAT_mesh-wants_pipeline-wanted";
export function pipelineWantedSubject(branch, source) {
return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source;
export function pipelineWantedSubject(branch, revision) {
return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision;
}
export const pipelineWantedSubjectAll = "mesh.seat.mesh-wants.event.pipeline-wanted.*.*";
@@ -0,0 +1,85 @@
package main
import (
"context"
"fmt"
"strconv"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-indexer/internal/index"
)
// entryAnswer is what indexer_entry answers: the trunk revision read, the entry's packages (the pins: each other-repository
// package with the version it builds against, empty while none is published), its named paths by artifact, its fingerprint
// and its manifest's digest.
type entryAnswer struct {
Module string `json:"module"`
Revision int64 `json:"revision"`
Packages map[string]string `json:"packages"`
Paths map[string][]string `json:"paths"`
Fp string `json:"fp,omitempty"`
ManifestDigest string `json:"manifest-digest,omitempty"`
}
// entryReadTool is indexer_entry (H-A-204; tools-are-how-a-module-is-spoken-to) over the indexer's catalog-revisions objects:
// one module's entry pins and paths at a trunk revision, small, so a pin can be read without the revision whole (1.3 MB,
// more than the bus carries).
func entryReadTool(o index.Objects) stdio.Tool {
return stdio.Tool{Name: index.EntryPinsTool,
Description: "Read one module's catalog graph entry pins: the packages it builds against with the version each is pinned to " +
"(an empty version while none is published), the paths each artifact is built from, its fingerprint and its manifest " +
"digest, at the newest trunk revision or at a trunk revision given; a small answer, for when the revision whole is too " +
"large for the bus (H-A-204).",
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module whose entry is read"},
"revision": map[string]any{"type": "integer", "description": "the trunk revision; none reads the newest"}}},
Run: func(args map[string]any) (any, error) {
module, _ := args["module"].(string)
module = strings.TrimSpace(module)
if module == "" {
return nil, fmt.Errorf("%s names the module", index.EntryPinsTool)
}
revision, err := revisionArg(args["revision"])
if err != nil {
return nil, err
}
rev, e, err := index.TrunkEntryAt(context.Background(), o, module, revision)
if err != nil {
return nil, err
}
out := entryAnswer{Module: e.Module, Revision: rev.Revision, Packages: e.Packages, Paths: map[string][]string{},
Fp: e.Fp, ManifestDigest: e.ManifestDigest}
if out.Packages == nil {
out.Packages = map[string]string{}
}
if e.Inputs != nil && e.Inputs.Paths != nil {
out.Paths = e.Inputs.Paths
}
return out, nil
}}
}
// revisionArg is the revision argument as a number: absent is 0 (the newest); a number or its digits as a string.
func revisionArg(v any) (int64, error) {
switch r := v.(type) {
case nil:
return 0, nil
case float64:
if r < 0 || r != float64(int64(r)) {
return 0, fmt.Errorf("revision %v is not a trunk revision's number", r)
}
return int64(r), nil
case string:
if strings.TrimSpace(r) == "" {
return 0, nil
}
n, err := strconv.ParseInt(strings.TrimSpace(r), 10, 64)
if err != nil || n < 0 {
return 0, fmt.Errorf("revision %q is not a trunk revision's number", r)
}
return n, nil
}
return 0, fmt.Errorf("revision %v is not a trunk revision's number", v)
}
@@ -0,0 +1,100 @@
package main
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-indexer/internal/index"
)
// memObjects is catalog-revisions in memory.
type memObjects map[string][]byte
func (m memObjects) Put(_ context.Context, name string, data []byte) error {
m[name] = data
return nil
}
func (m memObjects) Get(_ context.Context, name string) ([]byte, error) {
d, ok := m[name]
if !ok {
return nil, index.ErrNoObject
}
return d, nil
}
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
func (m memObjects) Names(context.Context) ([]string, error) {
var out []string
for n := range m {
out = append(out, n)
}
return out, nil
}
func (memObjects) Fill(context.Context) (int64, int64, error) { return 0, 0, nil }
// TestEntryTool: indexer_entry answers one entry's pins, paths, fingerprint and manifest digest, small, at the newest trunk
// revision and at a given one read from its kept object; an unknown module or revision is an error (H-A-204).
func TestEntryTool(t *testing.T) {
o := memObjects{}
keep := func(revision int64, entries ...meshcatalog.CatalogEntry) {
rev := meshcatalog.CatalogRevision{Branch: index.TrunkBranch, Revision: revision}
for _, e := range entries {
e.Manifest = map[string]any{"module": e.Module, "big": strings.Repeat("x", 64<<10)} // the manifest whole is never answered
data, _ := json.Marshal(e)
digest := e.Module + "-" + e.Fp
o[index.EntryObject(digest)] = data
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: digest})
}
data, _ := json.Marshal(rev)
o[index.BranchObject(index.TrunkBranch, revision)] = data
}
paths := &meshcatalog.EntryInputs{Paths: map[string][]string{"mesh-grants:program": {"modules/mesh-grants", "sdk"}}}
keep(7, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:old", ManifestDigest: "sha256:a", Inputs: paths,
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}}, meshcatalog.CatalogEntry{Module: "other", Fp: "src2:o"})
keep(9, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:new", ManifestDigest: "sha256:b", Inputs: paths,
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}})
tool := entryReadTool(o)
if tool.Name != "indexer_entry" {
t.Fatalf("name %s", tool.Name)
}
for _, c := range []struct {
name string
args map[string]any
want entryAnswer
err string
}{
{"newest", map[string]any{"module": "mesh-grants"}, entryAnswer{Module: "mesh-grants", Revision: 9, Fp: "src2:new", ManifestDigest: "sha256:b",
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}, Paths: paths.Paths}, ""},
{"given", map[string]any{"module": "mesh-grants", "revision": float64(7)}, entryAnswer{Module: "mesh-grants", Revision: 7, Fp: "src2:old", ManifestDigest: "sha256:a",
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}, Paths: paths.Paths}, ""},
{"given as digits, no paths", map[string]any{"module": "other", "revision": "7"}, entryAnswer{Module: "other", Revision: 7, Fp: "src2:o",
Packages: map[string]string{}, Paths: map[string][]string{}}, ""},
{"not in the newest", map[string]any{"module": "other"}, entryAnswer{}, "holds no module other"},
{"revision not kept", map[string]any{"module": "mesh-grants", "revision": float64(8)}, entryAnswer{}, "revision 8 is not kept"},
{"no module", map[string]any{}, entryAnswer{}, "names the module"},
} {
t.Run(c.name, func(t *testing.T) {
got, err := tool.Run(c.args)
if c.err != "" {
if err == nil || !strings.Contains(err.Error(), c.err) {
t.Fatalf("error %v, want %q", err, c.err)
}
return
}
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, c.want) {
t.Fatalf("answer %+v, want %+v", got, c.want)
}
data, _ := json.Marshal(got)
if len(data) >= 8<<10 {
t.Fatalf("answer of %d bytes, want under 8 KB", len(data))
}
})
}
}
@@ -193,7 +193,7 @@ func main() {
tools = append(tools, catalogTools(trunk.State, kept, watched)...)
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
// the parts whose entry declares it (novox/hq ADR 0663).
tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), entryReadTool(revisions), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}),
index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects))
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
@@ -433,7 +433,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
done()
done = step("following the assignments")
followAssignments(ctx, reader, seatHolders.holders(), process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) {
if err := ix.Recompute(ctx); err != nil {
if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil {
logger.Printf("writing the modules again after the assignments of a build module changed: %v", err)
}
}, logger.Printf)
@@ -449,7 +449,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// the toolchain.
return ix.ToolchainReported(ctx)
},
recompute: ix.Recompute}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger)
recompute: func(ctx context.Context) error { return ix.RecomputeKept(ctx, "the toolchains read") }}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger)
if err := tools.start(); err != nil {
return err
}
@@ -90,7 +90,7 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
}{
{name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"},
{name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"},
{name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1},
{name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.13","branch":"main","source":12,"revision":13,"outcome":"tier-late"}`, wanted: true, acked: 1},
{name: "not json", body: `nope`, termd: 1},
{name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"},
{name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"},
@@ -100,8 +100,8 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
switch {
case row.wanted:
heardPipelineWanted(context.Background(), ix, m, t.Logf)
if ix.wanted != "main.12 tier-late" {
t.Errorf("%s: was handed %q, want main.12 tier-late", row.name, ix.wanted)
if ix.wanted != "main.12.13 tier-late" { // its revision handed beside its source (H-A-214)
t.Errorf("%s: was handed %q, want main.12.13 tier-late", row.name, ix.wanted)
}
case row.ended:
heardPipelineEnded(context.Background(), ix, m, t.Logf)
@@ -169,8 +169,11 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte
return nil
}
func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64, outcome string) error {
func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source, revision int64, outcome string) error {
f.wanted = fmt.Sprintf("%s.%d %s", branch, source, outcome)
if revision > 0 {
f.wanted = fmt.Sprintf("%s.%d.%d %s", branch, source, revision, outcome)
}
return f.err
}
@@ -21,7 +21,7 @@ type runTaker interface {
StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error
EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error
HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error
HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error
HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error
JudgePins(ctx context.Context)
}
@@ -99,7 +99,7 @@ func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf fun
_ = m.Term()
return
}
err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Outcome)
err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Revision, e.Outcome)
settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf)
if err == nil {
ix.JudgePins(ctx)
@@ -22,8 +22,9 @@ var (
)
// heardVersion reads one message of the consumer as a version, false for any other subject. A package's is kept in its
// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives
// it again.
// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives it
// again. It recomputes no module: a fingerprint moves only with a kept catalog revision, a release's by the re-pin its
// package-published starts, one revision per module release (design 47 H-A-211, H-A-213 (2)).
func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) bool {
var h index.HeardVersion
switch seat, ok := publishedSeats[m.Subject()]; {
@@ -51,12 +52,6 @@ func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf
if kept {
logf("packages: %s %s kept in its series (published %v)", h.Package, h.Version, h.Published)
}
if kept && h.Published {
// A series' newest moved: its users' fingerprints take it (DLV-I6), in the same turn as the publish is heard.
if err := ix.Recompute(ctx); err != nil {
logf("writing the modules again after %s %s was published: %v", h.Package, h.Version, err)
}
}
_ = m.Ack()
return true
}
@@ -0,0 +1,52 @@
package main
import (
"context"
"encoding/json"
"errors"
"testing"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
"mesh-delivery-indexer/internal/index"
)
// ackedMsg is a message that counts its acknowledgements.
type ackedMsg struct {
jetstream.Msg
subject string
data []byte
acked int
}
func (a *ackedMsg) Subject() string { return a.subject }
func (a *ackedMsg) Data() []byte { return a.data }
func (a *ackedMsg) Ack() error { a.acked++; return nil }
// unlistedStore is a store whose listing fails: a recompute over it fails.
type unlistedStore struct{ memStore }
func (unlistedStore) Keys(context.Context) ([]string, error) {
return nil, errors.New("the bus does not answer")
}
// TestHeardVersionMovesNoFingerprint: a registry's published is kept in its series and acknowledged, and recomputes no
// module: a fingerprint moves only with a kept revision, a release's by its re-pin on the package-published (design 47
// H-A-211, H-A-213 (2); the pin specification's leftover L3). The modules state here fails any recompute.
func TestHeardVersionMovesNoFingerprint(t *testing.T) {
data, err := json.Marshal(gopackageregistry.PublishedEvent{Kind: "package", Name: "git.example/novox/lib", Version: "0.1.4"})
if err != nil {
t.Fatal(err)
}
ix := &index.Indexer{Packages: &memStore{vals: map[string][]byte{}}, Modules: &unlistedStore{memStore{vals: map[string][]byte{}}},
Manifests: &memStore{vals: map[string][]byte{}}}
m := &ackedMsg{subject: gopackageregistry.SubjectPublished, data: data}
if !heardVersion(context.Background(), ix, m, t.Logf) {
t.Fatal("the registry's published is not taken as a version")
}
if m.acked != 1 {
t.Errorf("acknowledged %d times, want once: a publish recomputes nothing that could fail", m.acked)
}
}
@@ -286,6 +286,14 @@ func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn {
return r
}
// TakeInAgain takes a repository in again at its kept merge commit (indexer_take_in_again, H-A-208, H-A-210), one take-in
// again at a time beside the held ones.
func (a actor) TakeInAgain(ctx context.Context, repository string) index.TakenIn {
a.s.again.Lock()
defer a.s.again.Unlock()
return a.Indexer.TakeInAgain(ctx, repository)
}
// HeldSkip ends a branch's held source revision unwritten (indexer_take_in_skip): take-in.taken says it skipped.
func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn {
r := index.TakenIn{Module: branch}
@@ -24,6 +24,8 @@ type takerIn interface {
// HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744).
HeldAgain(ctx context.Context, branch string) index.TakenIn
HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn
// TakeInAgain takes a repository in again at its kept merge commit (H-A-208, H-A-210).
TakeInAgain(ctx context.Context, repository string) index.TakenIn
}
// takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and
@@ -53,6 +55,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
} else if a.Held != "" {
asked, r = "take-in again of a held take-in", ix.HeldAgain(ctx, a.Held)
a.Module = a.Held
} else if a.Again != "" {
asked, r = "take-in again of a repository", ix.TakeInAgain(ctx, a.Again)
a.Module = a.Again
} else if a.Registry != nil {
asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a)
} else if a.Recompute {
@@ -6,6 +6,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"slices"
"strconv"
@@ -447,37 +448,42 @@ func TestIndexerWhole(t *testing.T) {
// The first revision: no mover, no pin; its entries are the base.
sayMoved(t, ctx, js, "main", 10, c10, true)
if first := heard.next(t, "main", 10); len(first.Entries) != 0 || first.Revision != 10 {
t.Fatalf("the first revision said %+v, want revision 10 with no entries", first)
// A revision's number is the indexer's own count, the branch's newest + 1 (design 47 H-A-213 (1)): the later take-ins of
// sources 11 to 13 are counted on from the number of source 10's.
first := heard.next(t, "main", 10)
r := func(source int64) int64 { return first.Revision + source - 10 }
rn := func(source int64) string { return fmt.Sprintf("main.%d", r(source)) }
if len(first.Entries) != 0 || first.Revision != 2 || first.Source != 10 {
t.Fatalf("the first revision said %+v, want revision 2 (the bed's seed is main.1, of source 0) of source 10 with no entries", first)
}
untilKey(t, ctx, pins, index.PinKey("main", 10), absent)
untilKey(t, ctx, pins, index.PinKey("main", r(10)), absent)
// One take-in, one revision carrying its movers, pinned for its pipeline; said again from the kept movers.
sayMoved(t, ctx, js, "main", 11, c11, false)
taken := heard.next(t, "main", 11)
if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Commit != c11 || taken.Entries[0].Fingerprint == "" ||
taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != 11 || taken.Call == "" {
taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != r(11) || taken.Call == "" {
t.Fatalf("take-in.taken of main 11 = %+v, want avahi at %s as its one mover", taken, c11)
}
rev, _, err := index.ReadRevision(ctx, objects, "main.11")
rev, _, err := index.ReadRevision(ctx, objects, rn(11))
if err != nil || len(rev.Movers) != 1 || rev.Movers[0] != taken.Entries[0] {
t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err)
}
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11"))
untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147
untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(11)))
untilKey(t, ctx, pins, index.PinKey("main", r(10)), pinned("pipeline."+rn(11))) // validate's base, H-A-147
info, err := objects.open(ctx)
if err != nil {
t.Fatal(err)
}
before, err := info.GetInfo(ctx, "main.11")
before, err := info.GetInfo(ctx, rn(11))
if err != nil {
t.Fatal(err)
}
sayMoved(t, ctx, js, "main", 11, c11, false)
if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != 11 {
if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != r(11) {
t.Errorf("the source-moved given again said %+v, want the kept movers %+v", again, taken.Entries)
}
if after, err := info.GetInfo(ctx, "main.11"); err != nil || after.NUID != before.NUID {
if after, err := info.GetInfo(ctx, rn(11)); err != nil || after.NUID != before.NUID {
t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err)
}
@@ -485,11 +491,11 @@ func TestIndexerWhole(t *testing.T) {
// (H-A-147), which the take-in pinned for it.
sayMoved(t, ctx, js, "main", 12, c12, false)
heard.next(t, "main", 12)
if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below
if err := pins.Delete(ctx, index.PinKey("main", r(12))); err != nil { // the take-in's own pin, so pipeline-started alone writes it below
t.Fatal(err)
}
sayPipeline(t, ctx, js, false, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
sayPipeline(t, ctx, js, false, "main", r(12))
untilKey(t, ctx, pins, index.PinKey("main", r(12)), pinned("pipeline."+rn(12)))
// A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174).
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
@@ -515,7 +521,7 @@ func TestIndexerWhole(t *testing.T) {
heard.acked(t, "main")
}
restart()
for _, name := range []string{"main.10", "main.11"} {
for _, name := range []string{rn(10), rn(11)} {
if _, _, err := index.ReadRevision(ctx, objects, name); err != nil {
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
}
@@ -553,27 +559,27 @@ func TestIndexerWhole(t *testing.T) {
t.Errorf("the held source-moved's consumer %+v (%v), want nothing unacknowledged", info, err)
}
setCap(t, ctx, js, 1<<30) // a larger cap alone wakes nothing: no timer takes a held take-in in
sayPipeline(t, ctx, js, true, "main", 11)
sayPipeline(t, ctx, js, true, "main", r(11))
if got := heard.next(t, "main", 13); len(got.Entries) != 1 || got.Entries[0].Commit != c13 {
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
}
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
// main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170).
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi")))
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(12), index.ReleaseRecord("main", "avahi")))
untilRevisionGone(t, ctx, js, rn(10)) // the end freed both of main.11's pins
// main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end),
// frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173).
sayPipeline(t, ctx, js, true, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool {
sayPipeline(t, ctx, js, true, "main", r(12))
untilKey(t, ctx, pins, index.PinKey("main", r(12)), func(v []byte) bool {
var p index.PinValue
return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi"))
})
wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning})
wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: rn(12), Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning})
if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil {
t.Fatal(err)
}
untilKey(t, ctx, pins, index.PinKey("main", 11), absent)
untilRevisionGone(t, ctx, js, "main.11")
untilKey(t, ctx, pins, index.PinKey("main", r(11)), absent)
untilRevisionGone(t, ctx, js, rn(11))
if n := len(heard.pending); n != 0 {
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
}
@@ -39,7 +39,9 @@ func checkCandidateModule(m meshcatalog.CandidatesByGroupValuePullsItemFpsValue)
return fmt.Errorf("index: the named path %q names no git object", p)
}
}
if err := sdkstep.PackageVersions(m.Packages).Check(); err != nil {
// An entry names a package with no published version yet with an empty version (design 47 H S8, H-A-204): it is checked
// as an entry's packages, not as an ask's.
if err := sdkstep.PackageVersions(m.Packages).CheckEntry(); err != nil {
return err
}
if m.Fp2 != "" {
@@ -0,0 +1,30 @@
package index
import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// TestCheckCandidateModulePins: an entry's package with no published version (an empty pin) passes a run on request's
// candidate; a pin that is no version is refused (design 47 H S8, the pin specification's item 4).
func TestCheckCandidateModulePins(t *testing.T) {
for _, c := range []struct {
name string
packages map[string]string
ok bool
}{
{"an empty pin", map[string]string{"git.example/novox/lib/go": ""}, true},
{"a version", map[string]string{"git.example/novox/lib/go": "v0.1.4"}, true},
{"no version", map[string]string{"git.example/novox/lib/go": "latest"}, false},
} {
t.Run(c.name, func(t *testing.T) {
m := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Fp2: "src2:" + strings.Repeat("ab", 32), Packages: c.packages}
err := checkCandidateModule(m)
if (err == nil) != c.ok {
t.Errorf("checkCandidateModule(%v) = %v, want ok %v", c.packages, err, c.ok)
}
})
}
}
@@ -36,13 +36,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
for _, n := range names {
have[n] = true
}
for _, want := range []string{"main.5", "feature/x=2ey.7", "feature/x=2ey.8"} {
for _, want := range []string{"main.1", "feature/x=2ey.1", "feature/x=2ey.2"} {
if !have[want] {
t.Errorf("the objects hold %v, want %s", names, want)
}
}
if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 8 {
t.Errorf("said %+v, want main 5, then the escaped branch 7 and 8", said)
if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 || said[2].Source != 8 {
t.Errorf("said %+v, want main 1 of source 5, then the escaped branch 1 and 2 of sources 7 and 8", said)
}
answer := func(branch string, revision int64) meshcatalog.RevisionAnswer {
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Revision: revision})
@@ -51,14 +51,14 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
}
return a
}
if a := answer("main", 0); a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" {
if a := answer("main", 0); a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" {
t.Errorf("main answered %+v", a)
}
if a := answer("feature/x=2ey", 0); a.Revision.Revision != 8 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" {
t.Errorf("the branch's newest answered %+v, want its revision 8 at c3", a.Revision)
if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" {
t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision)
}
if a := answer("feature/x=2ey", 7); a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's revision 7 answered %+v", a.Revision)
if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's revision 1 answered %+v", a.Revision)
}
if a := answer("beta", 0); !a.NoneYet {
t.Errorf("a branch with no revision answered %+v, want none-yet", a)
@@ -68,15 +68,22 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) {
return Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Source: source})
}
if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" {
if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" {
t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err)
}
if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's source 7 answered %+v (%v), want its revision 7 at c2", a.Revision, err)
if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's source 7 answered %+v (%v), want its revision 1 at c2", a.Revision, err)
}
if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 8 {
if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 {
t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err)
}
// H-A-213: a re-pin's revision of source 8 answers below the take-in's: source 8 answers revision 2.
if _, err := ix.keepBranchEntries(ctx, "feature/x.y", 8, releaseCall("mesh-sdk", "0.1.27"), tip("c4"), nil); err != nil {
t.Fatal(err)
}
if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 || a.Entries[0].Commit != "c3" {
t.Errorf("the branch's source 8 after a re-pin answered %+v (%v), want the take-in's revision 2", a.Revision, err)
}
if _, err := bySource("feature/x=2ey", 9); err == nil {
t.Error("a source no revision was taken in from was answered")
}
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"strings"
@@ -87,6 +88,37 @@ func BranchRevisionOf(branch, name string) (int64, bool) {
return r, err == nil && r > 0
}
// watchedBranches are the watched branches the bucket keeps a revision of, the first included when it keeps one, sorted:
// each name <branch>.<n> but an entry's, a candidate's and the earlier forms' (earlierForm).
func (ix *Indexer) watchedBranches(ctx context.Context) ([]string, error) {
if ix.Revisions == nil {
return nil, nil
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return nil, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
seen := map[string]bool{}
for _, n := range names {
if strings.HasPrefix(n, "entry.") || strings.HasPrefix(n, "candidate.") || earlierForm(n) {
continue
}
i := strings.LastIndexByte(n, '.')
if i <= 0 {
continue
}
if r, err := strconv.ParseInt(n[i+1:], 10, 64); err != nil || r <= 0 {
continue
}
branch, err := keys.UnescapeName(n[:i])
if err != nil {
continue
}
seen[branch] = true
}
return slices.Sorted(maps.Keys(seen)), nil
}
// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one.
func TrunkRevisionOf(name string) (int64, bool) { return BranchRevisionOf(TrunkBranch, name) }
@@ -210,6 +242,54 @@ func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Ca
}
}
// TrunkEntryAt is a module's entry in a trunk revision, with that revision's list (H-A-204): for revision 0 the newest trunk
// revision read as entryOf reads it (readNewestOf); else the kept list of that trunk revision and only the module's entry
// object, so one entry is read without the revision whole.
func TrunkEntryAt(ctx context.Context, o Objects, module string, revision int64) (meshcatalog.CatalogRevision, meshcatalog.CatalogEntry, error) {
if revision == 0 {
rev, entries, found, err := readNewestOf(ctx, o, TrunkBranch)
if err != nil {
return rev, meshcatalog.CatalogEntry{}, err
}
if !found {
return rev, meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
}
for _, e := range entries {
if e.Module == module {
return rev, e, nil
}
}
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", rev.Revision, module)
}
name := BranchObject(TrunkBranch, revision)
var rev meshcatalog.CatalogRevision
data, err := o.Get(ctx, name)
if errors.Is(err, ErrNoObject) {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d is not kept", revision)
}
if err != nil {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err)
}
if err := json.Unmarshal(data, &rev); err != nil {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err)
}
for _, ref := range rev.Entries {
if ref.Module != module {
continue
}
data, err := o.Get(ctx, EntryObject(ref.Digest))
if err != nil {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, module, ref.Digest, err)
}
var e meshcatalog.CatalogEntry
if err := json.Unmarshal(data, &e); err != nil {
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s: %w", name, module, err)
}
return rev, e, nil
}
return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", revision, module)
}
// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name.
func changedEntries(before, after []meshcatalog.EntryRef) []string {
was := map[string]string{}
@@ -266,17 +346,22 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata
}
// KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design
// 47 H §743, H-A-32): <branch>.<src>, the trunk's included, numbered by the source, and says catalog-revision naming the
// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline,
// pipeline.<branch>.<src>, in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written
// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it.
// 47 H §743, H-A-32), as keepBranchEntries does with no call.
func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error {
return ix.keepBranchEntries(ctx, branch, src, entries, movers)
_, err := ix.keepBranchEntries(ctx, branch, src, revisionCall{}, entries, movers)
return err
}
func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error {
// keepBranchEntries keeps entries as the next catalog revision of a watched branch, <branch>.<n>, the trunk's included: n is
// the indexer's own count, the branch's newest revision + 1, and src the gate's source revision it was read at, its field
// source (design 47 H-A-213 (1)); call is what made it (H-A-212, H-A-213 (3)): a take-in's the gate's call, a re-pin's
// "release of <module> <version>", a take-in again's "take-in again of <repository>". It says catalog-revision naming the
// branch, its source and its call. movers are kept on the revision (H-A-92), and the revision pinned for its one pipeline,
// pipeline.<branch>.<n>, in the same write when there is one (A-449a, H-A-213 (4)), with the branch's previous revision
// (H-A-147). It answers the number it kept; ErrObjectsFull is a store too full to keep it.
func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, call revisionCall, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) (int64, error) {
if ix.Revisions == nil {
return nil
return 0, nil
}
if branch == "" {
branch = TrunkBranch
@@ -286,28 +371,24 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
newest, _, err := newestOf(ctx, ix.Revisions, branch)
if errors.Is(err, process.ErrNotYet) {
ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects)
return err
return 0, err
}
if err != nil {
ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err)
return fmt.Errorf("the newest catalog graph revision: %w", err)
}
if _, err := ix.Revisions.Get(ctx, BranchObject(branch, src)); err == nil {
ix.logf("catalog graph: revision %s is kept already: not written again", BranchObject(branch, src))
return nil
} else if !errors.Is(err, ErrNoObject) {
return fmt.Errorf("revision %s: %w", BranchObject(branch, src), err)
return 0, fmt.Errorf("the newest catalog graph revision: %w", err)
}
number := newest.Revision + 1
entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208)
at := ix.now().UTC()
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: src, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries)
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: number, Kind: "trunk", Branch: branch, Source: src, Call: call.Call,
CallKind: call.Kind, Module: call.Module, Version: call.Version, Repository: call.Repository, At: &at}, entries)
if err != nil {
return err
return 0, err
}
rev.Movers = movers
// The base of the validate step's case before: the newest revision kept before this write, which the take-in pins for its
// pipeline below, named on the revision in this same write (H-A-147, H-A-151, H-A-154); none on the branch's first take-in.
pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 && newest.Revision != src
pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0
if pinsBase {
rev.Base = newest.Revision
}
@@ -319,32 +400,32 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull)
ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity)
ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused})
return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
return 0, fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
}
if err := ix.indexEntries(ctx, rev, entries); err != nil {
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
return err
return 0, err
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
return err
return 0, err
}
// The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a).
if len(movers) > 0 && ix.Pins != nil {
record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10)))
if err := ix.pin(ctx, PinKey(branch, src), record); err != nil {
ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err)
record := PipelineRecord(runner.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(number, 10)))
if err := ix.pin(ctx, PinKey(branch, number), record); err != nil {
ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, number), record, err)
return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, number), err)
}
// The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of
// the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it.
if pinsBase {
if err := ix.pin(ctx, PinKey(branch, rev.Base), record); err != nil {
ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err)
return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err)
}
} else {
ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record)
ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, number), record)
}
}
// The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it.
@@ -364,16 +445,48 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 {
ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity})
}
ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed))
ix.logf("catalog graph: revision %d of source revision %d kept (%s), %d entries, %d changed", rev.Revision, src, call.Call, len(rev.Entries), len(changed))
if ix.SayCatalogRevision != nil {
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
Branch: keys.EscapeName(branch), Source: src, Call: call.Call, CallKind: call.Kind, Module: call.Module, Version: call.Version,
Repository: call.Repository, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err)
}
}
return nil
return number, nil
}
// revisionCall is what made a catalog revision (design 47 H-A-213 (3), H-A-215): call, the words for a person; Kind, one of the
// seat mesh-catalog's three call kinds, which a reader reads instead of the words; Module and Version for a release, Repository
// for a take-in again. The zero value names no call (a revision kept by hand or a test).
type revisionCall struct {
Call, Kind, Module, Version, Repository string
}
// takeInCall is a take-in's: the gate's call its source revision names.
func takeInCall(call string) revisionCall {
return revisionCall{Call: call, Kind: meshcatalog.CatalogRevisionCallKindTakeIn}
}
// releaseCall is a re-pin's on the release of module at version (H-A-213 (2)).
func releaseCall(module, version string) revisionCall {
return revisionCall{Call: ReleaseCall(module, version), Kind: meshcatalog.CatalogRevisionCallKindRelease, Module: module, Version: version}
}
// takeInAgainCall is a take-in again's of repository (H-A-212).
func takeInAgainCall(repository string) revisionCall {
return revisionCall{Call: TakeInAgainCall(repository), Kind: meshcatalog.CatalogRevisionCallKindTakeInAgain, Repository: repository}
}
// recomputeCall is a recompute's that moved a fingerprint, its call naming the reason (H-A-217).
func recomputeCall(reason string) revisionCall {
return revisionCall{Call: RecomputeCall(reason), Kind: meshcatalog.CatalogRevisionCallKindRecompute}
}
// RecomputeCall is the call of the catalog revision a recompute keeps: its reason, such as the assignments of a build module,
// a toolchain or a hand act (design 47 H-A-217).
func RecomputeCall(reason string) string { return "recompute on " + reason }
// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held).
const HeldObjectsFull = "objects full"
@@ -21,6 +21,8 @@ const (
EntryIndexState = "entry-index"
// EntryIndexTool is the tool that reads a fingerprint's key and its object's digest (class read).
EntryIndexTool = "indexer_entries"
// EntryPinsTool is the tool that reads one module's entry pins and paths at a trunk revision (class read; H-A-204).
EntryPinsTool = "indexer_entry"
)
// EntryIndexValue is one key of entry-index: the digest of the entry object of the fingerprint the key names.
@@ -49,7 +49,7 @@ func TestAKeptRevisionIndexesItsEntries(t *testing.T) {
ctx := context.Background()
fp := "src2:" + strings.Repeat("d", 64)
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
t.Fatal(err)
}
if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" {
@@ -260,6 +260,32 @@ func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error)
}
return a.Result, nil
}},
{Name: TakeInAgainTool,
Description: "Take a repository in again now at the merge commit the indexer keeps of it (design 47 H-A-208, H-A-210): its " +
"module graph is read again (each module's requirements, transitively) and the modules state recomputed as a " +
"package's publish re-pins it, so a changed take-in rule reaches a repository with no merge since; pins and " +
"fingerprints move, no catalog revision is kept and the gate is not asked. An unknown repository is refused.",
Input: map[string]any{"type": "object", "required": []string{"repository"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "the repository as owner/name, such as novox/mesh-host"},
"why": map[string]any{"type": "string", "description": "why it is taken in again now"}}},
Run: func(args map[string]any) (any, error) {
repository := str(args, "repository")
if repository == "" {
return nil, errors.New("name the repository (repository) as owner/name")
}
data, err := json.Marshal(TakeInAsk{V: 1, Again: repository, Why: str(args, "why"), By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
a, err := ask("again."+keys.EscapeName(repository), data)
if err != nil {
return nil, err
}
if a.Result != nil && !a.Result.Stored {
return a.Result, fmt.Errorf("%s is not taken in again: %s", repository, a.Result.Answer)
}
return a.Result, nil
}},
{Name: TakeInSkipTool,
Description: "End a watched branch's held source-moved unwritten (design 47 H §744): it is acknowledged, take-in.taken is said " +
"with no entries and skipped, and the hand act is recorded through mesh-hand-acts before it is asked. indexer_pending " +
@@ -137,7 +137,7 @@ func TestTheEntryPinsEveryRequirementOfTheModuleGraph(t *testing.T) {
t.Errorf("packages = %+v, want the client's requirement and state's mesh-sdk, and no module of the repository", m.Inputs2)
}
builds := "git.novox.be/novox/mesh-delivery/modules/b/builds"
pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery")
pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery", nil)
if _, pinned := pins[builds]; pinned || len(pins) != 1 {
t.Errorf("pins = %v, want %s only, never a module of the repository", pins, conditions)
}
@@ -119,8 +119,9 @@ func ReleaseRecord(branch, module string) string {
// every revision of the branch below it; a mover the revision holds no release of (superseded, no version released) keeps its
// older release's revision. Outcome tier-late frees nothing: its later tiers' wants are not written, though their release
// records are held, and the same pipeline's later running frees them. A pipeline whose revision is not kept frees nothing and
// says so; one applied twice changes nothing.
func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error {
// says so; one applied twice changes nothing. The pipeline's revision is the one the event names (design 47 H-A-214); an event
// naming none (revision zero, said before the field) is of the source's lowest-numbered revision, the take-in's.
func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error {
if ix.Pins == nil || ix.Revisions == nil {
return errors.New("the indexer keeps no state " + PinsState)
}
@@ -128,12 +129,21 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc
ix.logf("pins: pipeline-wanted of %s %d is %s: it frees no release until its wants are written everywhere (H-A-174)", branch, source, outcome)
return nil
}
name, err := revisionOfSource(ctx, ix.Revisions, branch, source)
if err != nil {
ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err)
return nil
var name string
if revision > 0 {
name = RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: revision})
} else {
var err error
if name, err = revisionOfSource(ctx, ix.Revisions, branch, source); err != nil {
ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err)
return nil
}
}
rev, _, err := ReadRevision(ctx, ix.Revisions, name)
if revision > 0 && errors.Is(err, ErrNoObject) {
ix.logf("pins: pipeline-wanted of %s revision %d frees no release: %s is not kept", branch, revision, name)
return nil
}
if err != nil {
return fmt.Errorf("%s: %w", name, err)
}
@@ -106,42 +106,54 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
t.Fatal(err)
}
}
end := func(source int64, outcome, version string) {
endOf := func(source, revision int64, outcome, version string) {
t.Helper()
e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome,
e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", revision), Revision: revision, Source: source, Outcome: outcome,
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}}
if err := ix.HeardPins(ctx, nil, &e); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
}
end := func(source int64, outcome, version string) { t.Helper(); endOf(source, source, outcome, version) }
// An event naming no revision (said before the field) is of the source's lowest-numbered revision, the take-in's (H-A-214).
wanted := func(source int64, outcome string) {
t.Helper()
if err := ix.HeardPipelineWanted(ctx, "main", source, outcome); err != nil {
if err := ix.HeardPipelineWanted(ctx, "main", source, 0, outcome); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
}
take(10)
end(10, runner.PipelineEndedOutcomePassed, "0.1.10")
take(11)
end(11, runner.PipelineEndedOutcomeFailed, "")
want("a passed release past a later failed pipeline", "main.10", "main.11")
take(12)
end(12, runner.PipelineEndedOutcomePassed, "")
wanted(12, meshwants.PipelineWantedOutcomeRunning)
want("a passed pipeline releasing no version frees no older release", "main.10", "main.12")
take(13)
end(13, runner.PipelineEndedOutcomePassed, "0.1.13")
want("two releases, the unreleased one between them gone", "main.10", "main.13")
wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing
want("a tier-late pipeline-wanted", "main.10", "main.13")
wanted(13, meshwants.PipelineWantedOutcomeRunning)
wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more
want("the newer release written everywhere", "main.13")
if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
t.Errorf("the pin of main.13 holds %s, want its release alone", v)
take(4)
end(4, runner.PipelineEndedOutcomePassed, "0.1.10")
take(5)
end(5, runner.PipelineEndedOutcomeFailed, "")
want("a passed release past a later failed pipeline", "main.4", "main.5")
take(6)
end(6, runner.PipelineEndedOutcomePassed, "")
wanted(6, meshwants.PipelineWantedOutcomeRunning)
want("a passed pipeline releasing no version frees no older release", "main.4", "main.6")
take(7)
end(7, runner.PipelineEndedOutcomePassed, "0.1.13")
want("two releases, the unreleased one between them gone", "main.4", "main.7")
wanted(7, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing
want("a tier-late pipeline-wanted", "main.4", "main.7")
wanted(7, meshwants.PipelineWantedOutcomeRunning)
wanted(7, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more
want("the newer release written everywhere", "main.7")
if v, _, _, _ := ix.Pins.Get(ctx, "main.7"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
t.Errorf("the pin of main.7 holds %s, want its release alone", v)
}
// H-A-214: a re-pin's revision 8 of the same source 7 releases 0.1.14; its pipeline-wanted names revision 8, so the release
// on revision 8 frees the older one on revision 7, which the source's lowest-numbered revision would not.
take(7)
endOf(7, 8, runner.PipelineEndedOutcomePassed, "0.1.14")
want("a re-pin's release beside the take-in's", "main.7", "main.8")
if err := ix.HeardPipelineWanted(ctx, "main", 7, 8, meshwants.PipelineWantedOutcomeRunning); err != nil {
t.Fatal(err)
}
ix.JudgePins(ctx)
want("the re-pin's release written everywhere", "main.8")
}
func mustRecords(t *testing.T, v []byte) []string {
@@ -168,21 +180,22 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
before []int64 // sources taken in with no mover before the take-in of source 5
whileRuns []string
afterTheEnd []string
base int64 // the base main.5 names (H-A-154)
at int64 // the number the take-in of source 5 keeps: the branch's newest + 1 (H-A-213 (1))
base int64 // the base that revision names (H-A-154)
passed bool // main.5's pipeline passed: its release pins main.5
unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173)
then []int64 // sources taken in with no mover after the end
released []string
}{
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4},
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2},
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}},
{name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}},
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}},
{name: "the previous revision kept and pinned", before: []int64{3, 4}, at: 3, whileRuns: []string{"main.2", "main.3"}, afterTheEnd: []string{"main.3"}, base: 2},
{name: "the newest kept before, whatever its source", before: []int64{2}, at: 2, whileRuns: []string{"main.1", "main.2"}, afterTheEnd: []string{"main.2"}, base: 1},
{name: "no revision before it pins only its own", at: 1, whileRuns: []string{"main.1"}, afterTheEnd: []string{"main.1"}},
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"},
base: 1, passed: true, then: []int64{6}, afterTheEnd: []string{"main.2", "main.3"}, released: []string{"main.2"}},
{name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"},
base: 1, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.3"}},
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"},
base: 1, then: []int64{6}, afterTheEnd: []string{"main.3"}},
} {
t.Run(tc.name, func(t *testing.T) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
@@ -206,18 +219,19 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
keep(s, nil)
}
keep(5, mover)
if rev, _, err := ReadRevision(ctx, ix.Revisions, "main.5"); err != nil || rev.Base != tc.base {
t.Errorf("main.5 names the base %d (%v), want %d, the revision its take-in pinned (H-A-154)", rev.Base, err, tc.base)
at := fmt.Sprintf("main.%d", tc.at)
if rev, _, err := ReadRevision(ctx, ix.Revisions, at); err != nil || rev.Base != tc.base || rev.Source != 5 {
t.Errorf("%s names the source %d and the base %d (%v), want 5 and %d, the revision its take-in pinned (H-A-154)", at, rev.Source, rev.Base, err, tc.base)
}
if got := kept(); !slices.Equal(got, tc.whileRuns) {
t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns)
}
for _, k := range tc.whileRuns {
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) {
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline." + at}) {
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
}
}
end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed,
end := runner.PipelineEnded{Branch: "main", Pipeline: at, Revision: tc.at, Outcome: runner.PipelineEndedOutcomeFailed,
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}}
if tc.unreleased {
end.Movers[0].Version = ""
@@ -0,0 +1,100 @@
package index
import (
"context"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Design 47 H-A-204, H-A-201, H-A-165: an entry's packages name every requirement of its module graph of another
// repository: one the form's versions leave out (an import form naming only what the build imported) is pinned by its
// series' newest published version, and one none publishes yet is kept with no version, never left out.
func TestAnEntryNamesEveryRequirementPinnedOrWaiting(t *testing.T) {
const (
wireguard = "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go"
busgrants = "git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants"
sdk = "git.novox.be/novox/mesh-sdk/go"
)
series := map[string]SeriesPackage{"wg": {Name: "mesh-wireguard/go", GoModule: wireguard, Series: catalogue.DefaultSeries,
Newest: &meshcatalog.PackagesByPackageSeriesValueNewest{Version: "0.1.47"}}}
pins := releasedPins(sdkstep.PackageVersions{sdk: "c0ffee"}, series, catalogue.Manifest{}, "novox/mesh-catalog", []string{busgrants, sdk, wireguard})
want := sdkstep.PackageVersions{sdk: "", wireguard: "v0.1.47", busgrants: ""}
if len(pins) != len(want) {
t.Fatalf("pins = %v, want %v", pins, want)
}
for p, v := range want {
if got, named := pins[p]; !named || got != v {
t.Errorf("the pin of %s is %q (named %v), want %q", p, got, named, v)
}
}
}
// Design 47 H-A-204, H-A-165: a release re-pins every kept entry naming one of its packages, of any repository and
// changed or not, by its pins or by its kept requirements alone: here g, of another repository, holds no pin of mesh-sdk.
// H-A-211, H-A-212, H-A-213: on every watched branch holding g, the re-pin keeps the branch's next revision, of the same
// source as the one before it, with the call "release of mesh-sdk 0.1.27" and g as its mover at its new fingerprint, and
// says catalog-revision with that call.
func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
ctx := context.Background()
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
ix, mods, _ := ptpIndexer(t, objects)
ix.Revisions, ix.Packages = newMemObjects(1<<20), newMem()
reportToolchain(t, ix, "mesh-build-go", "n1", "go", "1.26.0")
var said []meshcatalog.CatalogRevisionEvent
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { said = append(said, e); return nil }
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil)
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
g, _ := moduleOf(t, mods, "g")
if g.Inputs2 == nil || !slices.Contains(g.Inputs2.Packages, "git.novox.be/novox/mesh-sdk/go") {
t.Fatalf("g = %+v, want mesh-sdk among its kept requirements", g.Inputs2)
}
g.Packages = nil
mods.put("g", mustEncode(t, g))
mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40),
Path: meshcatalogclient.RootPath, None2: None2Pending}))
entries, err := ix.viewEntries(ctx)
if err != nil {
t.Fatal(err)
}
sources := map[string]int64{"main": 5, "feature/x": 9}
for branch, src := range sources {
if err := ix.KeepBranchEntries(ctx, branch, src, entries, nil); err != nil {
t.Fatal(err)
}
}
said = nil
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.27", Published: true}); err != nil {
t.Fatal(err)
}
if n, err := ix.Repin(ctx, "mesh-sdk", "0.1.27"); err != nil || n != 2 {
t.Errorf("a release of mesh-sdk re-pinned %d modules (%v), want g on each of the two branches, which requires it with no pin", n, err)
}
now, _ := moduleOf(t, mods, "g")
if now.Fp2 == "" || now.Fp2 == g.Fp2 || now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.27" {
t.Fatalf("g after the re-pin = %s pinned %v, want a new fingerprint pinned to v0.1.27 (was %s)", now.Fp2, now.Packages, g.Fp2)
}
for branch, src := range sources {
rev, kept, found, err := readNewestOf(ctx, ix.Revisions, branch)
if err != nil || !found || rev.Revision != 2 || rev.Source != src || rev.Call != "release of mesh-sdk 0.1.27" ||
rev.CallKind != meshcatalog.CatalogRevisionCallKindRelease || rev.Module != "mesh-sdk" || rev.Version != "0.1.27" || // H-A-215
len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != now.Fp2 {
t.Errorf("%s's newest = %+v (%v), want revision 2 of source %d, its call the release, g its mover at %s", branch, rev, err, src, now.Fp2)
continue
}
if i := slices.IndexFunc(kept, func(e meshcatalog.CatalogEntry) bool { return e.Module == "g" }); i < 0 || kept[i].Fp != now.Fp2 {
t.Errorf("%s's newest holds no entry of g at %s", branch, now.Fp2)
}
}
if len(said) != 2 || said[0].Call != "release of mesh-sdk 0.1.27" || said[1].Call != said[0].Call ||
said[0].CallKind != meshcatalog.CatalogRevisionEventCallKindRelease || said[1].Module != "mesh-sdk" || said[1].Version != "0.1.27" {
t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said)
}
}
@@ -272,7 +272,7 @@ func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn
r.Answer = "the newest catalog graph revision cannot be read: " + err.Error()
return r
}
if err := ix.Recompute(ctx); err != nil {
if err := ix.RecomputeKept(ctx, fmt.Sprintf("a hand act by %s (%s)", by, why)); err != nil {
r.Answer = "the recompute failed: " + err.Error()
return r
}
@@ -167,18 +167,24 @@ func revisionOfSource(ctx context.Context, o Objects, branch string, source int6
}
}
slices.SortFunc(found, func(a, b int64) int { return int(b - a) })
// A re-pin or a take-in again keeps a later revision of the same source (H-A-211, H-A-213): the source's own revision is
// the lowest numbered of it, the take-in's.
lowest := ""
for _, r := range found {
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: r})
rev, _, err := ReadRevision(ctx, o, name)
if err != nil {
return "", err
}
if rev.Source == source {
return name, nil
}
if rev.Source < source {
break
}
if rev.Source == source {
lowest = name
}
}
if lowest != "" {
return lowest, nil
}
return "", fmt.Errorf("no revision of %q was taken in from the source revision %d", branch, source)
}
@@ -0,0 +1,78 @@
package index
import (
"context"
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/process"
)
// TakeInAgainTool takes a repository in again at its kept merge commit (design 47 H-A-208, H-A-210): a rule change of the
// take-in (such as the walk of the module graph, H-A-201) reaches a repository with no merge since.
const TakeInAgainTool = "indexer_take_in_again"
// TakeInAgainCall is the call of the catalog revision a take-in again keeps (design 47 H-A-212, H-A-213 (3)).
func TakeInAgainCall(repository string) string { return "take-in again of " + repository }
// TakeInAgain re-reads the repository's module graph at the merge commit its taken key keeps (wholeAt, the reading of every
// take-in: the kept requirements of each of its modules, H-A-201, H-A-157) and applies it as a take-in does, whose recompute
// of the modules state is the one a publish re-pin runs (recomputeLocked): pins and fingerprints move as they would there.
// It keeps the result as the trunk's next catalog revision of the same source, its call TakeInAgainCall (H-A-212, H-A-213),
// and asks no gate. A repository with no taken key is refused, said in the answer.
func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn {
repository = strings.TrimSpace(repository)
r := TakenIn{Module: repository, Repository: repository}
refuse := func(format string, a ...any) TakenIn {
r.Answer = "refused: " + fmt.Sprintf(format, a...)
ix.logf("take-in again of %s: %s", repository, r.Answer)
return r
}
if ix.Taken == nil || ix.Modules == nil {
return refuse("the indexer keeps no taken merges or no modules, so nothing is taken in again")
}
key, err := trunk.RepositoryKey(repository)
if err != nil {
return refuse("%q names no repository as owner/name", repository)
}
value, _, found, err := ix.Taken.Get(ctx, key)
if errors.Is(err, process.ErrNotYet) {
return refuse("the state taken is not on the bus yet")
}
if err != nil {
return refuse("taken.%s cannot be read: %v", key, err)
}
if !found {
return refuse("the indexer keeps no merge of %s (no key taken.%s): an unknown repository, or one never taken in", repository, key)
}
t, err := takein.DecodeTaken(value)
if err != nil || t.Merge == "" {
return refuse("taken.%s names no merge commit", key)
}
ix.pendingMu.Lock() // one take-in step at a time (pending.go)
defer ix.pendingMu.Unlock()
f, err := ix.wholeAt(ctx, repository, t.Merge)
if err != nil {
return refuse("%v", err)
}
f.Branch = TrunkBranch
applied, err := ix.ApplyMessage(ctx, f)
if err != nil {
return refuse("taking %s at %.12s in again: %v", repository, t.Merge, err)
}
// The take-in again keeps its revision as a re-pin does: the branch's next, of the same source, its call naming the
// repository, its movers every module whose fingerprint moved (design 47 H-A-212, H-A-213).
if err := ix.keepRepinned(ctx, ix, TrunkBranch, takeInAgainCall(repository), nil); err != nil {
return refuse("taken in again at %.12s, but its catalog revision is not kept: %v", t.Merge, err)
}
r.Stored, r.Commit = true, t.Merge
r.Answer = fmt.Sprintf("taken in again at %.12s: %d modules written", t.Merge, len(applied.Written))
if len(applied.Written) > 0 {
r.Answer += " (" + strings.Join(applied.Written, ", ") + ")"
}
ix.logf("take-in again of %s: %s", repository, r.Answer)
return r
}
@@ -0,0 +1,113 @@
package index
import (
"context"
"maps"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// Design 47 H-A-208, H-A-210: a repository taken in before the walk of the module graph (H-A-201) keeps its modules' direct
// requirements only; indexer_take_in_again re-reads it at its kept merge commit and recomputes the modules state, so the
// module's kept requirements and pins gain the transitive package, and keeps the trunk's next catalog revision of the same
// source with the call "take-in again of <repository>" (H-A-212, H-A-213). An unknown repository is refused.
func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) {
const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
ix, mods, _ := ptpIndexer(t, objects)
ix.Taken, ix.Revisions, ix.Packages = newMem(), newMemObjects(1<<20), newMem()
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
direct := map[string]string{"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n"}
walked := map[string]string{"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n"}
goMods := direct // before the rule: the client's go.mod reads as requiring nothing
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
if c, ok := goMods[dir]; ok {
return []byte(c), nil
}
if dir == "modules/c/client" {
return []byte("module git.novox.be/novox/mesh-delivery/modules/c/client\n"), nil
}
return []byte(ptpGoMods[dir]), nil
}
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram},
map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}})
var readAt []string
ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) {
readAt = append(readAt, repository+"@"+sha)
return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil
}
ctx := context.Background()
for p, v := range map[string]string{conditions: "0.1.4", "git.novox.be/novox/mesh-sdk/go": "0.1.27"} {
if _, err := ix.HearVersion(ctx, HeardVersion{Package: p, GoModule: p, Version: v, Published: true}); err != nil {
t.Fatal(err)
}
}
// the modules that hold the two packages, so their published versions pin them (fingerprint.HolderOf)
mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40),
Path: meshcatalogclient.RootPath, None2: None2Pending}))
mods.put("meshconditionsclient", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-contracts", Commit: strings.Repeat("2", 40),
Path: "health-and-repair/go/clients/meshconditionsclient", None2: None2Pending}))
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
before, _ := moduleOf(t, mods, "g")
if before.Inputs2 == nil || slices.Contains(before.Inputs2.Packages, conditions) || strings.Contains(before.None2, conditions) {
t.Fatalf("before: packages = %+v (%s), want no transitive requirement yet", before.Inputs2, before.None2)
}
entries, err := ix.viewEntries(ctx)
if err != nil {
t.Fatal(err)
}
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 4, entries, nil); err != nil {
t.Fatal(err)
}
if r := ix.TakeInAgain(ctx, "novox/unknown"); r.Stored || !strings.HasPrefix(r.Answer, "refused:") {
t.Errorf("unknown repository: %+v, want refused", r)
}
goMods = maps.Clone(direct)
maps.Copy(goMods, walked)
r := ix.TakeInAgain(ctx, "novox/mesh-delivery")
if !r.Stored || r.Commit != merge {
t.Fatalf("take-in again = %+v, want stored at the kept merge %s", r, merge)
}
if len(readAt) == 0 || readAt[0] != "novox/mesh-delivery@"+merge {
t.Errorf("read at %v, want the kept merge commit", readAt)
}
m, _ := moduleOf(t, mods, "g")
if m.Inputs2 == nil || !slices.Contains(m.Inputs2.Packages, conditions) {
t.Errorf("after: kept requirements = %+v, want %s", m.Inputs2, conditions)
}
// The recompute pins it at its published version, which moves the fingerprint (H-A-167, H-A-204), and the take-in again
// keeps the trunk's next revision of the same source, its call naming the repository, g its mover (H-A-212, H-A-213).
if m.Fp2 == "" || m.Fp2 == before.Fp2 || m.Packages[conditions] != "v0.1.4" {
t.Errorf("after: g = %s (%s) pinned %v, want a new fingerprint pinned to v0.1.4 (was %s)", m.Fp2, m.None2, m.Packages, before.Fp2)
}
rev, _, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil || !found || rev.Revision != 2 || rev.Source != 4 || rev.Call != "take-in again of novox/mesh-delivery" ||
rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeInAgain || rev.Repository != "novox/mesh-delivery" || // H-A-215
len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != m.Fp2 {
t.Errorf("the newest trunk revision = %+v (%v), want revision 2 of source 4 with the take-in again's call and g its mover", rev, err)
}
// H-A-217: a recompute that moves no fingerprint keeps nothing; one that moves one (a toolchain reported) keeps the trunk's
// next revision of the same source, its call-kind recompute, its call naming the reason, the moved modules its movers.
if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil {
t.Fatal(err)
}
if rev, _, _, _ := readNewestOf(ctx, ix.Revisions, TrunkBranch); rev.Revision != 2 {
t.Errorf("a recompute that moved nothing kept revision %d (%s), want none", rev.Revision, rev.Call)
}
reportToolchainRecipe(t, ix, "mesh-build-go", "novox", "go", "go1.27.1", "2") // a recipe change moves a Go module at once
moved, _ := moduleOf(t, mods, "g")
rev, _, found, err = readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil || !found || rev.Revision != 3 || rev.Source != 4 || rev.CallKind != meshcatalog.CatalogRevisionCallKindRecompute ||
rev.Call != RecomputeCall("a toolchain reported") || !slices.ContainsFunc(rev.Movers, func(e meshcatalog.TakeInEntry) bool {
return e.Module == "g" && e.Fingerprint == moved.Fp2 && moved.Fp2 != m.Fp2
}) {
t.Errorf("after a toolchain reported: the newest trunk revision = %+v (%v), want revision 3 of source 4, kind recompute, g its mover at %s (was %s)",
rev, err, moved.Fp2, m.Fp2)
}
}
@@ -46,6 +46,8 @@ type TakeInAsk struct {
HeldSource int64 `json:"held-source,omitempty"`
Skip bool `json:"skip,omitempty"`
NamedMember string `json:"named-member,omitempty"`
// Again names the repository taken in again at its kept merge commit (indexer_take_in_again, take_in_again.go; H-A-208, H-A-210).
Again string `json:"again,omitempty"`
// RunDrop asks one run on request, by its group run-<id>, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194).
RunDrop string `json:"run-drop,omitempty"`
// Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations,
@@ -2,9 +2,11 @@ package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -25,9 +27,10 @@ func TakenID(branch string, source int64) string {
// TakeInSource takes one source revision of a watched branch in, as the gate's source-moved names it (design 47 H §743): for each
// repository it moves, the trees at the named commit through the seat git (wholeAt) and the fingerprints of the modules
// registered for it; its movers (moversOf) against the branch's newest catalog revision; one catalog revision of the branch,
// <branch>.<source>, carrying the movers and pinned for the take-in's pipeline when it has one (KeepBranchEntries); and
// the branch's next, <branch>.<n> of source <source>, carrying the movers and pinned for the take-in's pipeline when it has one
// (keepBranchEntries, H-A-213); and
// take-in.taken once, with branch, source, revision, call and one entry per mover; a branch's first revision (start) has no
// mover (§768). A revision already kept under that name (a
// mover (§768). A revision already kept of that source (a
// restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched
// branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision.
// An error leaves the source-moved held, written into the state held behind its branch's earlier ones (ErrHeldUnwritten when
@@ -57,13 +60,13 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
name := BranchObject(e.Branch, e.Source)
if kept, _, err := ReadRevision(ctx, ix.Revisions, name); err == nil {
taken.Revision, taken.Entries = kept.Revision, kept.Movers
ix.logf("take-in: %s is kept already: take-in.taken said again from its %d movers", name, len(kept.Movers))
return taken, ix.sayTaken(ctx, taken)
} else if !errors.Is(err, ErrNoObject) {
// A source revision kept already is found by the revision's field source, never by its number (H-A-213 (1)).
if kept, found, err := takenOfSource(ctx, ix.Revisions, e.Branch, e.Source); err != nil {
return taken, err
} else if found {
taken.Revision, taken.Entries = kept.Revision, kept.Movers
ix.logf("take-in: source revision %d of %s is kept already as %s: take-in.taken said again from its %d movers", e.Source, e.Branch, RevisionObject(kept), len(kept.Movers))
return taken, ix.sayTaken(ctx, taken)
}
_, before, _, err := readNewestOf(ctx, ix.Revisions, e.Branch)
if err != nil {
@@ -106,14 +109,15 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE
if e.Start {
movers = nil // a branch's first revision moves nothing: its entries are the base later take-ins are held against (§768, H-A-19)
}
if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil {
number, err := ix.keepBranchEntries(ctx, e.Branch, e.Source, takeInCall(e.Call), entries, movers)
if err != nil {
if errors.Is(err, ErrObjectsFull) {
ix.heldFull.Store(true) // room is said once the store has room again (RoomAgain), and wakes the take-in (§744)
}
return taken, err
}
taken.Revision, taken.Entries = e.Source, movers
ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, name, len(e.Heads), len(movers))
taken.Revision, taken.Entries = number, movers
ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, BranchObject(e.Branch, number), len(e.Heads), len(movers))
return taken, ix.sayTaken(ctx, taken)
}
@@ -157,22 +161,57 @@ func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelive
if ix.Revisions == nil {
return e, false, errors.New("the indexer keeps no catalog revisions")
}
names, err := ix.Revisions.Names(ctx)
// The gate's source a branch is taken in at is its newest revision's field source, never the revision's number (H-A-213 (1)).
newest, found, err := newestOf(ctx, ix.Revisions, branch)
if err != nil {
return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
return e, false, fmt.Errorf("the newest catalog revision of %s: %w", branch, err)
}
var kept int64
for _, n := range names {
if r, ok := BranchRevisionOf(branch, n); ok && r > kept {
kept = r
}
}
if kept >= a.Revision {
if found && newest.Source >= a.Revision {
return e, false, nil
}
e.Start = kept == 0
e.Start = !found
for _, r := range a.Repositories {
e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head})
}
return e, true, nil
}
// takenOfSource is the revision a take-in of source src kept on branch: of the branch's revisions whose field source is src,
// the lowest numbered (a re-pin or a take-in again keeps the same source on a later number, H-A-211, H-A-213). The revisions
// are read from the newest down while their source is not below src; none is false.
func takenOfSource(ctx context.Context, o Objects, branch string, src int64) (meshcatalog.CatalogRevision, bool, error) {
names, err := o.Names(ctx)
if err != nil {
return meshcatalog.CatalogRevision{}, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
var numbers []int64
for _, n := range names {
if r, ok := BranchRevisionOf(branch, n); ok {
numbers = append(numbers, r)
}
}
slices.Sort(numbers)
var kept meshcatalog.CatalogRevision
found := false
for i := len(numbers) - 1; i >= 0; i-- {
name := BranchObject(branch, numbers[i])
data, err := o.Get(ctx, name)
if errors.Is(err, ErrNoObject) {
continue // removed under the read by the pin judge
}
if err != nil {
return kept, false, fmt.Errorf("revision %s: %w", name, err)
}
var rev meshcatalog.CatalogRevision
if err := json.Unmarshal(data, &rev); err != nil {
return kept, false, fmt.Errorf("%s: %w", name, err)
}
if rev.Source < src {
break
}
if rev.Source == src {
kept, found = rev, true
}
}
return kept, found, nil
}
@@ -43,7 +43,8 @@ func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTa
return ix, objects, said, f
}
// Design 47 H §743, H-A-32, H-A-92: a take-in writes one catalog revision <branch>.<source> carrying its movers, pins it for the
// Design 47 H §743, H-A-32, H-A-92, H-A-213 (1): a take-in writes one catalog revision <branch>.<n>, n the branch's newest + 1
// and its source the gate's, carrying its movers, pins it for the
// take-in's pipeline and says take-in.taken once; a redelivery writes nothing again and says the same; a revision with no mover
// pins nothing and says no entries; the old form is refused.
func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
@@ -56,19 +57,20 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
t.Fatal(err)
}
if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Fingerprint == "" || taken.Entries[0].Commit != f.Merge ||
taken.Revision != 7 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call {
t.Fatalf("taken %+v, want revision 7 of main with avahi as its one mover", taken)
taken.Revision != 1 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call {
t.Fatalf("taken %+v, want revision 1 (the branch's first) of source 7 of main with avahi as its one mover", taken)
}
rev, _, err := ReadRevision(ctx, objects, "main.7")
if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 {
t.Fatalf("main.7 = %+v (%v), want it kept carrying the movers", rev, err)
rev, _, err := ReadRevision(ctx, objects, "main.1")
if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 ||
rev.Call != moved.Call || rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeIn { // H-A-215
t.Fatalf("main.1 = %+v (%v), want it kept carrying the movers", rev, err)
}
if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 7)); !found {
t.Error("main.7 is not pinned for its pipeline")
if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 1)); !found {
t.Error("main.1 is not pinned for its pipeline")
} else {
var pv PinValue
if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.7"}) {
t.Errorf("main.7 is pinned by %s, want pipeline.main.7", v)
if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.1"}) {
t.Errorf("main.1 is pinned by %s, want pipeline.main.1", v)
}
}
puts := objects.puts
@@ -79,14 +81,14 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
same := moved
same.Source, same.Call = 8, "fedcba9876543210"
none, err := ix.TakeInSource(ctx, same)
if err != nil || len(none.Entries) != 0 || none.Revision != 8 {
t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 8 with no entries", none, err)
if err != nil || len(none.Entries) != 0 || none.Revision != 2 || none.Source != 8 {
t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 2 of source 8 with no entries", none, err)
}
if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 8)); found {
if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 2)); found {
t.Error("a take-in with no mover pinned its revision")
}
if _, _, err := ReadRevision(ctx, objects, "main.8"); err != nil {
t.Errorf("main.8 is not kept: %v", err)
if _, _, err := ReadRevision(ctx, objects, "main.2"); err != nil {
t.Errorf("main.2 is not kept: %v", err)
}
if len(*said) != 3 {
t.Errorf("said %d take-in.taken, want 3", len(*said))
@@ -79,9 +79,22 @@ func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value
return ix.ToolchainReported(ctx)
}
// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute).
// ToolchainReported is what a changed toolchain moves: the modules written again, kept as a revision (RecomputeKept).
func (ix *Indexer) ToolchainReported(ctx context.Context) error {
return ix.Recompute(ctx)
return ix.RecomputeKept(ctx, "a toolchain reported")
}
// RecomputeKept is Recompute kept as a re-pin is (design 47 H-A-217): when it moved at least one fingerprint against the
// trunk's newest revision, it keeps the trunk's next revision of the same source, its movers the modules that moved, its
// call-kind recompute and its call naming reason, pinned for its pipeline, and says catalog-revision; one that moves none
// keeps nothing. Recompute writes the trunk's modules state, so only the trunk's revision is kept.
func (ix *Indexer) RecomputeKept(ctx context.Context, reason string) error {
ix.pendingMu.Lock() // one take-in step at a time, so the trunk's newest stays the one compared against (pending.go)
defer ix.pendingMu.Unlock()
if err := ix.Recompute(ctx); err != nil {
return err
}
return ix.keepRepinned(ctx, ix, TrunkBranch, recomputeCall(reason), nil)
}
// agreed is the toolchain of each language every reporting node agrees on, or why there is none.
@@ -298,7 +311,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
r.fp, r.none = f.fp, f.none
if r.fp != "" {
// a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository)
r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository, kept.Packages)
r.tcs = toolchainsUsed(toolchains)
}
}
@@ -517,7 +530,7 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2,
if f.none != "" || f.fp == "" {
return "", nil, false
}
fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository)
fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository, kept.Packages)
return fp, pkgs, fp == k.held.Fp2
}
@@ -544,7 +557,7 @@ func (ix *Indexer) RecomputeAtStart(ctx context.Context) error {
ix.logf("no toolchain is read yet: the new-form fingerprints are computed when one is")
return nil
}
return ix.Recompute(ctx)
return ix.RecomputeKept(ctx, "the indexer's start")
}
// fillPaths gives every module whose key names no directory (a key written before the indexer learned paths) the one its
@@ -598,29 +611,128 @@ func importsOf(pkgs sdkstep.PackageVersions, repository string) bool {
return false
}
// ReleaseCall is the call of the catalog revision a re-pin keeps: the release of a module at a version (design 47 H-A-213 (2)).
func ReleaseCall(module, version string) string { return "release of " + module + " " + version }
// Repin moves the pins of every module whose packages hold a package of module (a held module's directory holds it) to
// module's newest released version, the pass of H-A-161 for those modules alone: a version-released of module starts it
// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It answers how many it named.
func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) {
// module's newest released version, the pass of H-A-161 for those modules alone: a release of module at version starts it
// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It runs on every watched branch
// holding such a module: the first on the indexer's own modules state, another on its newest revision's picture (as a
// take-in does), and keeps on each whose re-pinned fingerprints moved the branch's next catalog revision, of the same
// source as its newest, with the call ReleaseCall and those modules as its movers, pinned for the release's pipeline, and
// says catalog-revision; no take-in.taken (H-A-211, H-A-212, H-A-213). It answers how many modules it named.
func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, error) {
if ix.Modules == nil || ix.Manifests == nil {
return 0, nil
}
held, err := ix.heldModules(ctx)
ix.pendingMu.Lock() // one take-in step at a time, so the branch's newest stays the one re-pinned (pending.go)
defer ix.pendingMu.Unlock()
branches, err := ix.watchedBranches(ctx)
if err != nil {
return 0, err
}
if !slices.Contains(branches, TrunkBranch) {
branches = append([]string{TrunkBranch}, branches...) // the modules state is re-pinned whether a revision is kept or not
}
named := 0
for _, branch := range branches {
x := ix
if !isTrunkBranch(branch) {
view, nodes, _, err := BranchView(ctx, ix.Revisions, branch)
if err != nil {
return named, err
}
d := ix.dry()
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) }
x = d.ix
}
only, err := x.repinned(ctx, module)
if err != nil {
return named, err
}
if len(only) == 0 {
continue
}
named += len(only)
x.recompute.Lock()
err = x.recomputeLocked(ctx, nil, nil, "", only)
x.recompute.Unlock()
if err != nil {
return named, err
}
if err := ix.keepRepinned(ctx, x, branch, releaseCall(module, version), only); err != nil {
return named, err
}
}
return named, nil
}
// repinned are the modules the indexer's modules state holds whose packages name a package of module: by their pins, and
// by their kept requirements, which name a package they hold no pin of yet (design 47 H-A-204, H-A-165, H-A-167).
func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool, error) {
held, err := ix.heldModules(ctx)
if err != nil {
return nil, err
}
only := map[string]bool{}
for key, m := range held {
for p := range m.Packages {
names := slices.Collect(maps.Keys(m.Packages))
if m.Inputs2 != nil {
names = append(names, m.Inputs2.Packages...)
}
for _, p := range names {
if moduleOfKey(fingerprint.HolderOf(p, held)) == module {
only[key] = true
}
}
}
if len(only) == 0 {
return 0, nil
return only, nil
}
// keepRepinned keeps x's view, re-pinned for the modules only names (every module for nil), as branch's next catalog revision
// of the source of its newest, with call, when a fingerprint of them moved against that newest; its movers are those that
// moved (H-A-211, H-A-212, H-A-213). A branch with no revision kept yet keeps none: there is no source to keep it of.
func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch string, call revisionCall, only map[string]bool) error {
if ix.Revisions == nil {
return nil
}
ix.recompute.Lock()
defer ix.recompute.Unlock()
return len(only), ix.recomputeLocked(ctx, nil, nil, "", only)
newest, before, found, err := readNewestOf(ctx, ix.Revisions, branch)
if err != nil {
return fmt.Errorf("the newest catalog revision of %s: %w", branch, err)
}
if !found {
ix.logf("re-pin (%s): %s keeps no revision yet, so none is kept for it", call.Call, branch)
return nil
}
entries, err := x.viewEntries(ctx)
if err != nil {
return err
}
was := make(map[string]string, len(before))
for _, e := range before {
was[e.Module] = e.Fp
}
var movers []meshcatalog.TakeInEntry
moved := slices.Sorted(maps.Keys(only))
if only == nil { // a take-in again or a recompute: every module whose fingerprint moved (H-A-212, H-A-217)
moved = slices.Sorted(maps.Keys(entries))
}
for _, key := range moved {
e, ok := entries[key]
if !ok || e.Fp == "" || was[e.Module] == e.Fp {
continue
}
movers = append(movers, meshcatalog.TakeInEntry{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
}
if len(movers) == 0 {
ix.logf("re-pin (%s): no fingerprint of %s moved, so no revision is kept", call.Call, branch)
return nil
}
number, err := ix.keepBranchEntries(ctx, branch, newest.Source, call, entries, movers)
if err != nil {
return err
}
ix.logf("re-pin (%s): %s kept as revision %d of source %d, %d movers", call.Call, branch, number, newest.Source, len(movers))
return nil
}
@@ -504,7 +504,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
}
return first.KeepBranchEntries(context.Background(), TrunkBranch, source, entries, nil)
}
if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 6 ||
if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 ||
firstSaid[0].Entries != int64(len(builderModules(t))) {
t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err)
}
@@ -519,7 +519,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
if err := takeIn(7); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) {
t.Errorf("the next take-in after a write said %+v (%v), want revision 7 naming avahi", firstSaid, err)
}
if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 7 {
if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 {
t.Errorf("the view at a restart is read at %d (%v), want 7", at, err)
} else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") {
t.Errorf("the view at a restart lost the verdict: %s", raw)
@@ -536,21 +536,21 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
t.Errorf("the first-view tools %+v serve no modules", FirstViewTools)
}
if a, err := views.FirstView(testSelf, ModulesState, trunkStates.State(ModulesState), trunkStates.State(views.MarksState), "", time.Now()); err != nil ||
a.Revision != 7 || len(a.Values) != len(builderModules(t)) {
t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 7", len(a.Values), a.Revision, err)
a.Revision != 2 || len(a.Values) != len(builderModules(t)) {
t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 2", len(a.Values), a.Revision, err)
} else if m, err := meshcatalogclient.DecodeModule(a.Values[viewKey]); err != nil || m.Manifest == "" {
t.Errorf("the first view's avahi = %s (%v), want the module's value naming its manifest", a.Values[viewKey], err)
}
if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":7`) {
t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 7", e, ok, err)
if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) {
t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err)
}
// A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never
// one revision read per key (about 11 s a page live, 2026-11-14).
reads := 0
keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}}
if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil ||
a.Revision != 7 || len(a.Values) != len(builderModules(t)) || a.Next != "" {
t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 7 in one page", len(a.Values), a.Revision, a.Next, err)
a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" {
t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err)
}
if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 {
t.Errorf("the kept view read the revision %d times (%v), want once", reads, err)
@@ -822,7 +822,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.2.0", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil {
t.Fatal(err)
}
if _, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil { // the pass for avahi alone: mesh-tools is held stale here
if _, err := ix.Repin(context.Background(), "mesh-sdk", "0.1.27"); err != nil { // the pass for avahi alone: mesh-tools is held stale here
t.Fatal(err)
}
if b, _ := moduleOf(t, mods, "avahi"); b.Fp2 != a.Fp2 || b.Packages[sdk] != "v0.1.18" {
@@ -376,12 +376,20 @@ func isFloor(raw []byte) bool {
// release number is never a pin. A package with none published in that series is kept with no version, so its kind-1
// edge stands (H §746) and its build waits for the publish (H-A-165). A module of the module's own repository is never
// pinned, whatever a build reported: it is built from the tree (design 47 H-A-198, H-A-201, H-A-202). nil for none.
func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string) sdkstep.PackageVersions {
if len(versions) == 0 {
//
// required are the mesh packages the module's go.mod graph requires (its kept packages, H-A-201): each of another
// repository is named whether or not the form's versions name it, with no version while none is published, never left
// out, so the builder waits for its publish instead of the build module refusing (design 47 H-A-204, H-A-165).
func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) sdkstep.PackageVersions {
names := slices.Collect(maps.Keys(versions))
if repository != "" {
names = append(names, fingerprint.OtherPackages(required, repository)...)
}
if len(names) == 0 {
return nil
}
out := sdkstep.PackageVersions{}
for p := range versions {
for _, p := range names {
if repo, _, ok := trunk.PackageRepository(p); ok && repository != "" && trunk.SameRepository(repo, repository) {
continue
}
@@ -403,8 +411,8 @@ func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPack
// form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover
// built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that
// stands is kept while its pins are the ones it was built with, else its import form takes the new pins.
func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, series, m, repository)
func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) (string, sdkstep.PackageVersions) {
pins := releasedPins(f.versions, series, m, repository, required)
if f.stands {
if maps.Equal(f.built, map[string]string(pins)) {
return f.fp, pins
@@ -443,7 +451,7 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions {
// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error.
func (ix *Indexer) repin(ctx context.Context, module, version string) error {
n, err := ix.Repin(ctx, module)
n, err := ix.Repin(ctx, module, version)
if err != nil {
return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err)
}
@@ -102,7 +102,7 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn(
n := NewNamed(newMem(), nil)
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }}
n.Changed = ix.RemoveUnlisted
if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
t.Fatal(err)
}
if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { // the revision that listed it is removed
@@ -213,7 +213,7 @@ func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testin
fp := namedFp("c")
setup := func(n *Named) (*Indexer, func() bool) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }}
if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
t.Fatal(err)
}
if err := ix.Revisions.Delete(ctx, "main.1"); err != nil {
@@ -205,9 +205,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
index.FingerprintsAgainTool, // issue 1966
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || !slices.Contains(m.Tools, "modules") {
index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
@@ -57,6 +57,7 @@
"indexer_recompute",
"indexer_drop_candidate_graph",
"indexer_entries",
"indexer_entry",
"indexer_named_drop",
"indexer_proposals",
"indexer_proposal_drop",
@@ -66,6 +67,7 @@
"indexer_pin_drop",
"indexer_pending",
"indexer_take_in_pending",
"indexer_take_in_again",
"indexer_take_in_skip",
"indexer_runs",
"indexer_run_drop"
@@ -206,7 +206,7 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error)
if raw == nil {
return seat.Pipeline{}, false, nil
}
p, err := seat.DecodePipelinesByBranchSourceValue(raw)
p, err := seat.DecodePipelinesByBranchRevisionValue(raw)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err)
}
@@ -50,7 +50,7 @@ type Begin struct {
// Key is a take-in's pipeline key in the state pipelines, `<branch>.<source>`, the branch escaped as a key's name is
// (design 47 H §746).
func Key(branch string, source int64) string {
return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10))
return seat.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10))
}
// PublishedKey is the key of a pipeline a package's publish starts, `<branch>.<source>.published.<package>.<version>`,
@@ -40,7 +40,7 @@ func TestStarts(t *testing.T) {
if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Movers: []seat.Mover{{Module: "app"}}}); err != nil {
t.Fatal(err)
}
if b, r, ok := seat.ParsePipelinesByBranchSourceKey(Key("release.1", 3)); !ok || r != "3" || b == "release" {
if b, r, ok := seat.ParsePipelinesByBranchRevisionKey(Key("release.1", 3)); !ok || r != "3" || b == "release" {
t.Errorf("the key of release.1 is %q", Key("release.1", 3))
}
if _, err := s.Start(Begin{Branch: "main", Movers: []seat.Mover{{Module: "app"}}}); err == nil {