diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 0ca00286b..2d1604792 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -169,7 +169,7 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { changed := false var kept []tierMover defer func() { - if err := e.startTiers(p.Pipeline, kept); err != nil { + if err := e.startTiers(p.Pipeline, p.Source, kept); err != nil { e.logf("the tier progress of %s is not written: %v", p.Pipeline, err) } }() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go index 5211d5641..5c45c24f4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go @@ -42,11 +42,14 @@ type tierMover struct { Tier int64 `json:"tier"` } -// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, and the movers. +// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, the movers, and the source +// revision its pipeline-ended names, said in pipeline-wanted beside the key's revision (design 47 H-A-214, H-A-216); zero in a +// value kept before the field, read as the key's number. type tierProgress struct { Tier int64 `json:"tier"` Since time.Time `json:"since"` Movers []tierMover `json:"movers"` + Source int64 `json:"source,omitempty"` } // tierKey is the key of pipeline's tier progress; pipeline is the runner's key `.`, its branch escaped. @@ -60,7 +63,7 @@ func isTierKey(key string) bool { // startTiers writes the tier progress of a passed pipeline whose movers' versions are now kept, its first tier the lowest of // theirs, since now. A pipeline heard again writes nothing. -func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error { +func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMover) error { if len(movers) == 0 { return nil } @@ -68,7 +71,7 @@ func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error { for _, m := range movers { first = min(first, m.Tier) } - value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers}) + value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers, Source: source}) if err != nil { return err } @@ -131,7 +134,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st continue } pipeline := strings.TrimPrefix(key, tierPrefix) - branchEsc, source, _ := strings.Cut(pipeline, ".") + branchEsc, number, _ := strings.Cut(pipeline, ".") branch, err := keys.UnescapeName(branchEsc) if err != nil { continue @@ -144,7 +147,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st notRunning := p.notRunning(n, placedOn, pipeline, branch, t, t.Tier) late := !now.Before(t.Since.Add(bound)) if late { - p.tierLate(pipeline, branch, source, t, notRunning) + p.tierLate(pipeline, branch, number, t, notRunning) break } if len(notRunning) > 0 { @@ -165,10 +168,10 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st p.logf("the tier progress %s is not deleted: %v", key, err) break } - p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeRunning, nil) + p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeRunning, nil) break } - value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers}) + value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers, Source: t.Source}) if err == nil { err = p.ends.st.swap(key, value, rev) } @@ -217,7 +220,7 @@ func (p *passer) notRunning(n assign.Nodes, placedOn func(module, branch string) // tierLate ends a pipeline's delivery at a tier past tier-bound: the finding tier-late is raised and pipeline-wanted says tier-late, // once each; the finding clears once the late movers run or a later pipeline moved them. The progress stays, so no later tier is // written, until no later tier's mover is kept at this pipeline any more. -func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRunning []meshwants.NotRunning) { +func (p *passer) tierLate(pipeline, branch, number string, t tierProgress, notRunning []meshwants.NotRunning) { o := tierLateFinding(pipeline, t.Tier, notRunning) key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind) if len(notRunning) > 0 { @@ -231,7 +234,7 @@ func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRu p.said[key+".raised"] = true delete(p.said, key+".cleared") p.logf("pipeline %s: tier %d is late (tier-bound %s): %s; no later tier is written", pipeline, t.Tier, p.tierBoundOf(), o.Summary) - p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeTierLate, notRunning) + p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeTierLate, notRunning) } } else if !p.said[key+".cleared"] { if p.findings.Delete != nil { @@ -271,11 +274,16 @@ func tierLateFinding(pipeline string, tier int64, notRunning []meshwants.NotRunn Source: "assigner." + kindTierLate} } -// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome). -func (p *passer) sayPipelineWanted(pipeline, branch, source, outcome string, notRunning []meshwants.NotRunning) { - var src int64 - _, _ = fmt.Sscan(source, &src) - body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: src, Outcome: outcome, NotRunning: notRunning} +// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome): its +// revision the number of the pipeline's key `.`, its source the one the pipeline-ended named (design 47 +// H-A-214, H-A-216), the key's number for a progress kept before it held one. +func (p *passer) sayPipelineWanted(pipeline, branch, number string, source int64, outcome string, notRunning []meshwants.NotRunning) { + var revision int64 + _, _ = fmt.Sscan(number, &revision) + if source <= 0 { + source = revision + } + body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: source, Revision: revision, Outcome: outcome, NotRunning: notRunning} event := meshwants.EventPipelineWanted + "." + pipeline if err := p.wantsSay(event, body, "pipeline-wanted."+pipeline+"."+outcome); err != nil { p.logf("pipeline-wanted of %s is not said: %v", pipeline, err) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go index 39d9bb398..9937f7754 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go @@ -62,8 +62,11 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { p.w.mu.Unlock() p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now } - if _, err := p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed), - mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))); err != nil { + // The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216). + e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed), + mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed)) + e.Source = 4 + if _, err := p.ends.take(e); err != nil { t.Fatal(err) } wanted := func(node, module string) bool { @@ -88,7 +91,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web}) _ = p.w.hearRunning("running.g14.web", raw) p.wants(context.Background()) - if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || len(findings) != 0 { + if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 { t.Errorf("every tier runs: said %+v, findings %v", said, findings) } if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok { @@ -108,8 +111,14 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") { t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings) } - if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" { - t.Errorf("pipeline-wanted tier-late is not said once: %+v", said) + if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" || + said[0].Revision != 5 || said[0].Source != 4 { + t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said) + } + // A tier progress kept before it held its source says the key's number as both (H-A-216). + p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil) + if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 { + t.Errorf("a progress with no source: said %+v, want revision and source 5", said) } if wanted("g14", "web") || wanted("ace", "web") { t.Error("a consumer's want is written after its provider's tier was late") diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go index 261c6f668..1ad5e63d5 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go @@ -715,6 +715,10 @@ type CatalogRevision struct { Base int64 `json:"base,omitempty"` // the watched branch the revision is of Branch string `json:"branch,omitempty"` + // what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's + Call string `json:"call,omitempty"` + // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's + CallKind string `json:"call-kind,omitempty"` // the seat definitions it was judged with, each by its digest, read whole through seat-definitions Definitions []CatalogRevisionDefinitionsItem `json:"definitions,omitempty"` // one per module, by name @@ -725,8 +729,12 @@ type CatalogRevision struct { Heads string `json:"heads,omitempty"` // trunk or candidate Kind string `json:"kind"` + // for call-kind release, the module released (H-A-215) + Module string `json:"module,omitempty"` // the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover Movers []TakeInEntry `json:"movers,omitempty"` + // for call-kind take-in-again, the repository taken in again (H-A-215) + Repository string `json:"repository,omitempty"` // the catalog graph revision's number, counted by its holder Revision int64 `json:"revision"` // the source revision it was taken in from; for a candidate, the source revision its heads are merged onto @@ -735,8 +743,18 @@ type CatalogRevision struct { Trunk []Head `json:"trunk,omitempty"` // the layout version V int64 `json:"v,omitempty"` + // for call-kind release, the version released (H-A-215) + Version string `json:"version,omitempty"` } +// The values of CatalogRevision's call-kind. +const ( + CatalogRevisionCallKindTakeIn = "take-in" + CatalogRevisionCallKindRelease = "release" + CatalogRevisionCallKindTakeInAgain = "take-in-again" + CatalogRevisionCallKindRecompute = "recompute" +) + // Edge is one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses). type Edge struct { // declared in the artifact entry's needs of A's manifest, where nothing else says it @@ -1301,6 +1319,10 @@ type CatalogRevisionEvent struct { At *time.Time `json:"at,omitempty"` // the watched branch the revision is of, escaped as a key name is Branch string `json:"branch,omitempty"` + // what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's + Call string `json:"call,omitempty"` + // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's + CallKind string `json:"call-kind,omitempty"` // the modules whose entries differ from the revision before it of the same kind Changed []string `json:"changed,omitempty"` // how many entries it lists @@ -1308,12 +1330,26 @@ type CatalogRevisionEvent struct { Group string `json:"group,omitempty"` Heads string `json:"heads,omitempty"` // trunk or candidate: a candidate's moves no running part - Kind string `json:"kind"` - Revision int64 `json:"revision"` - Source int64 `json:"source"` - V int64 `json:"v"` + Kind string `json:"kind"` + // for call-kind release, the module released (H-A-215) + Module string `json:"module,omitempty"` + // for call-kind take-in-again, the repository taken in again (H-A-215) + Repository string `json:"repository,omitempty"` + Revision int64 `json:"revision"` + Source int64 `json:"source"` + V int64 `json:"v"` + // for call-kind release, the version released (H-A-215) + Version string `json:"version,omitempty"` } +// The values of CatalogRevisionEvent's call-kind. +const ( + CatalogRevisionEventCallKindTakeIn = "take-in" + CatalogRevisionEventCallKindRelease = "release" + CatalogRevisionEventCallKindTakeInAgain = "take-in-again" + CatalogRevisionEventCallKindRecompute = "recompute" +) + // FillingEvent is the body of the event filling. type FillingEvent = Fill diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go index e8f85fa88..5258e5ea3 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go @@ -17,7 +17,7 @@ func TestFixtures(t *testing.T) { decode func([]byte) (any, error) subjectIs *regexp.Regexp }{ - {name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", + {name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}", decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")}, {name: "a candidate's revision kept", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")}, @@ -59,7 +59,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v ModulesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "the module verb (captured: xclip)", subject: "", given: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", wire: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", decode: func(b []byte) (any, error) { var v ModuleAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, - {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", + {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}", decode: func(b []byte) (any, error) { var v CatalogRevisionsByBranchRevisionValue err := json.Unmarshal(b, &v) diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index 058625fb4..8b5f340e5 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -65,9 +65,9 @@ const ( // StreamPipelineStarted is the stream pipeline-started is kept in. const StreamPipelineStarted = "SEAT_mesh-pipeline-runner_pipeline-started" -// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... -func SubjectPipelineStarted(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source +// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... +func SubjectPipelineStarted(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision } // SubjectPipelineStartedAll hears every pipeline-started. @@ -76,9 +76,9 @@ const SubjectPipelineStartedAll = "mesh.seat.mesh-pipeline-runner.event.pipeline // StreamPipelineEnded is the stream pipeline-ended is kept in. const StreamPipelineEnded = "SEAT_mesh-pipeline-runner_pipeline-ended" -// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... -func SubjectPipelineEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source +// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... +func SubjectPipelineEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision } // SubjectPipelineEndedAll hears every pipeline-ended. @@ -120,9 +120,9 @@ const SubjectStepEndedAll = "mesh.seat.mesh-pipeline-runner.event.step-ended.*.* // StreamWaitHeld is the stream wait-held is kept in. const StreamWaitHeld = "SEAT_mesh-pipeline-runner_wait-held" -// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... -func SubjectWaitHeld(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source +// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... +func SubjectWaitHeld(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision } // SubjectWaitHeldAll hears every wait-held. @@ -131,9 +131,9 @@ const SubjectWaitHeldAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*" // StreamWaitEnded is the stream wait-ended is kept in. const StreamWaitEnded = "SEAT_mesh-pipeline-runner_wait-ended" -// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... -func SubjectWaitEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source +// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... +func SubjectWaitEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision } // SubjectWaitEndedAll hears every wait-ended. @@ -156,11 +156,11 @@ const ( StateStepOutputs = "step-outputs" ) -// PipelinesByBranchSourceKey is the key of one value of the state pipelines under the form ".". -func PipelinesByBranchSourceKey(branch, source string) string { return branch + "." + source } +// PipelinesByBranchRevisionKey is the key of one value of the state pipelines under the form ".". +func PipelinesByBranchRevisionKey(branch, revision string) string { return branch + "." + revision } -// ParsePipelinesByBranchSourceKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. -func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { +// ParsePipelinesByBranchRevisionKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. +func ParsePipelinesByBranchRevisionKey(key string) (string, string, bool) { rest := key v0, after0, found0 := strings.Cut(rest, ".") if !found0 || v0 == "" { @@ -178,9 +178,9 @@ func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { return v0, v2, true } -// DecodePipelinesByBranchSourceValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchSourceValue does not know and any text after the value. -func DecodePipelinesByBranchSourceValue(raw []byte) (PipelinesByBranchSourceValue, error) { - var v PipelinesByBranchSourceValue +// DecodePipelinesByBranchRevisionValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchRevisionValue does not know and any text after the value. +func DecodePipelinesByBranchRevisionValue(raw []byte) (PipelinesByBranchRevisionValue, error) { + var v PipelinesByBranchRevisionValue dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&v); err != nil { @@ -670,7 +670,7 @@ type Pipeline struct { // the modules the pipeline moves, its take-in's entries (design 47 H §746) Movers []Mover `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) + // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -720,7 +720,7 @@ type PipelineEnded struct { // per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) Movers []MoverEnded `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -753,7 +753,7 @@ type PipelineStarted struct { Call string `json:"call,omitempty"` // the pipeline's movers Movers []Mover `json:"movers"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -814,7 +814,7 @@ type StepEvent struct { // the module's build-input fingerprint at the commit Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` Result StepResult `json:"result"` // the catalog graph revision of the branch the pipeline runs for @@ -966,7 +966,7 @@ type ToolchainUsed struct { Version string `json:"version"` } -// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72). +// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221). type Wait struct { Artifact string `json:"artifact,omitempty"` // the step's own ask, where it has one (design 47 G §731) @@ -983,7 +983,7 @@ type Wait struct { Module string `json:"module,omitempty"` // what the step waits on (design 47 *Health and going back*) On string `json:"on"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -1024,7 +1024,7 @@ const ( // Waiting is a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72). type Waiting struct { - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` } @@ -1355,8 +1355,8 @@ type FullEvent = Fill // RoomEvent is the body of the event room. type RoomEvent = Fill -// PipelinesByBranchSourceValue is one value of the state pipelines under the key .. -type PipelinesByBranchSourceValue = Pipeline +// PipelinesByBranchRevisionValue is one value of the state pipelines under the key .. +type PipelinesByBranchRevisionValue = Pipeline // PipelinesByBranchSourcePublishedPackageVersionValue is one value of the state pipelines under the key ..published... type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go index 23b59e9cd..4aaa6d53c 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go @@ -45,7 +45,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a take-in's pipeline kept, its steps naming their mover", subject: "", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, @@ -73,13 +73,13 @@ func TestFixtures(t *testing.T) { }, subjectIs: nil}, {name: "a pipeline waiting its turn, its seq given", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, {name: "a waiting pipeline names the movers a newer take-in took out of it", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go index a4dc8aa90..0df94fede 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go @@ -83,9 +83,9 @@ const SubjectWantedAll = "mesh.seat.mesh-wants.event.wanted.*.*" // StreamPipelineWanted is the stream pipeline-wanted is kept in. const StreamPipelineWanted = "SEAT_mesh-wants_pipeline-wanted" -// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted... -func SubjectPipelineWanted(branch, source string) string { - return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source +// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted... +func SubjectPipelineWanted(branch, revision string) string { + return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision } // SubjectPipelineWantedAll hears every pipeline-wanted. @@ -1001,10 +1001,12 @@ type PipelineWanted struct { // for tier-late, the late tier's movers not running and the machines they do not run on NotRunning []NotRunning `json:"not-running,omitempty"` Outcome string `json:"outcome"` - // the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746) + // the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4)) Pipeline string `json:"pipeline"` - Source int64 `json:"source"` - V int64 `json:"v"` + // the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event + Revision int64 `json:"revision,omitempty"` + Source int64 `json:"source"` + V int64 `json:"v"` } // The values of PipelineWanted's outcome. diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go index daeed530f..d623d162e 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go @@ -29,7 +29,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a want removed, its node left or its module unplaced", subject: "mesh.seat.mesh-wants.event.wanted.n1.xclip", given: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}", decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")}, - {name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}", + {name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}", decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a pipeline's tier late, naming the movers and machines not running", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}", wire: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}", decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")}, diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json index fd3678dfd..1c0b4dd0a 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json @@ -8,6 +8,10 @@ "revision": 12, "kind": "trunk", "source": 3589, + "call": "release of mesh-sdk 0.1.27", + "call-kind": "release", + "module": "mesh-sdk", + "version": "0.1.27", "entries": 2, "changed": [ "xclip" diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json index e537ca821..965b8ff63 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json @@ -209,7 +209,11 @@ "repository": "novox/mesh-catalog" } ], - "base": 9 + "base": 9, + "call": "release of mesh-sdk 0.1.27", + "call-kind": "release", + "module": "mesh-sdk", + "version": "0.1.27" } }, { diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json index e9db48fce..164a7576f 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json @@ -657,6 +657,32 @@ "source": { "type": "integer" }, + "call": { + "type": "string", + "description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string" }, @@ -1871,6 +1897,32 @@ "type": "integer", "description": "the source revision it was taken in from; for a candidate, the source revision its heads are merged onto" }, + "call": { + "type": "string", + "description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string", "description": "a candidate's group" diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json index cb8363f23..9229229fb 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json @@ -1,7 +1,7 @@ [ { "name": "a take-in's pipeline kept, its steps naming their mover", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3589", "branch": "main", @@ -200,7 +200,7 @@ }, { "name": "a pipeline waiting its turn, its seq given", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", @@ -224,7 +224,7 @@ }, { "name": "a waiting pipeline names the movers a newer take-in took out of it", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index 5bf14a52c..fd84b289c 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -672,7 +672,7 @@ "events": [ { "name": "pipeline-started", - "subject": "pipeline-started..", + "subject": "pipeline-started..", "body": { "$ref": "#/defs/PipelineStarted" }, @@ -683,7 +683,7 @@ }, { "name": "pipeline-ended", - "subject": "pipeline-ended..", + "subject": "pipeline-ended..", "body": { "$ref": "#/defs/PipelineEnded" }, @@ -727,7 +727,7 @@ }, { "name": "wait-held", - "subject": "wait-held..", + "subject": "wait-held..", "body": { "$ref": "#/defs/Wait" }, @@ -738,7 +738,7 @@ }, { "name": "wait-ended", - "subject": "wait-ended..", + "subject": "wait-ended..", "body": { "$ref": "#/defs/Wait" }, @@ -778,11 +778,11 @@ "description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)", "key-forms": [ { - "key": ".", + "key": ".", "value": { "$ref": "#/defs/Pipeline" }, - "description": "a take-in's pipeline" + "description": "a pipeline of one catalog revision, keyed by the revision's number (design 47 H-A-213 (4), H-A-219); one kept from before under its source has the same form" }, { "key": "..published..", @@ -1199,7 +1199,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" + "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" }, "branch": { "type": "string", @@ -1313,7 +1313,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1388,7 +1388,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1495,7 +1495,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string" @@ -1550,7 +1550,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string", @@ -1635,7 +1635,7 @@ "description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)" } }, - "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72)", + "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221)", "required": [ "v", "pipeline", @@ -1978,7 +1978,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" } }, "description": "a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72)", diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json b/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json index feeace4ed..3381500a1 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json @@ -65,12 +65,13 @@ { "name": "a pipeline's wants all run", "of": "event:pipeline-wanted", - "subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589", + "subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", "given": { "v": 1, - "pipeline": "main.3589", + "pipeline": "main.3590", "branch": "main", "source": 3589, + "revision": 3590, "outcome": "running" } }, diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json index f09ae1f9e..b827436e9 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json @@ -434,7 +434,7 @@ }, { "name": "pipeline-wanted", - "subject": "pipeline-wanted..", + "subject": "pipeline-wanted..", "keep": { "max-age-days": 30, "max-mib": 64 @@ -1725,7 +1725,7 @@ }, "pipeline": { "type": "string", - "description": "the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746)" + "description": "the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4))" }, "branch": { "type": "string" @@ -1733,6 +1733,10 @@ "source": { "type": "integer" }, + "revision": { + "type": "integer", + "description": "the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event" + }, "outcome": { "type": "string", "enum": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts index b3d0c6975..1bef6feb8 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts @@ -277,6 +277,10 @@ export interface CatalogRevision { "base"?: number; /** the watched branch the revision is of */ "branch"?: string; + /** what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's */ + "call"?: string; + /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's */ + "call-kind"?: "take-in" | "release" | "take-in-again" | "recompute"; /** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */ "definitions"?: CatalogRevisionDefinitionsItem[]; /** one per module, by name */ @@ -287,8 +291,12 @@ export interface CatalogRevision { "heads"?: string; /** trunk or candidate */ "kind": string; + /** for call-kind release, the module released (H-A-215) */ + "module"?: string; /** the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover */ "movers"?: TakeInEntry[]; + /** for call-kind take-in-again, the repository taken in again (H-A-215) */ + "repository"?: string; /** the catalog graph revision's number, counted by its holder */ "revision": number; /** the source revision it was taken in from; for a candidate, the source revision its heads are merged onto */ @@ -297,6 +305,8 @@ export interface CatalogRevision { "trunk"?: Head[]; /** the layout version */ "v"?: number; + /** for call-kind release, the version released (H-A-215) */ + "version"?: string; } /** one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses) */ @@ -796,6 +806,10 @@ export interface CatalogRevisionEvent { "at"?: string; /** the watched branch the revision is of, escaped as a key name is */ "branch"?: string; + /** what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's */ + "call"?: string; + /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's */ + "call-kind"?: "take-in" | "release" | "take-in-again" | "recompute"; /** the modules whose entries differ from the revision before it of the same kind */ "changed"?: string[]; /** how many entries it lists */ @@ -804,9 +818,15 @@ export interface CatalogRevisionEvent { "heads"?: string; /** trunk or candidate: a candidate's moves no running part */ "kind": string; + /** for call-kind release, the module released (H-A-215) */ + "module"?: string; + /** for call-kind take-in-again, the repository taken in again (H-A-215) */ + "repository"?: string; "revision": number; "source": number; "v": number; + /** for call-kind release, the version released (H-A-215) */ + "version"?: string; } /** is the body of the event filling. */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index 748152b9f..5c11adad5 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -40,13 +40,13 @@ export declare const family: { readonly name: "mesh-pipeline-runner.events"; rea export declare const pipelineStartedStream: "SEAT_mesh-pipeline-runner_pipeline-started"; -export declare function pipelineStartedSubject(branch: string, source: string): string; +export declare function pipelineStartedSubject(branch: string, revision: string): string; export declare const pipelineStartedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export declare const pipelineEndedStream: "SEAT_mesh-pipeline-runner_pipeline-ended"; -export declare function pipelineEndedSubject(branch: string, source: string): string; +export declare function pipelineEndedSubject(branch: string, revision: string): string; export declare const pipelineEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -70,13 +70,13 @@ export declare const stepEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event. export declare const waitHeldStream: "SEAT_mesh-pipeline-runner_wait-held"; -export declare function waitHeldSubject(branch: string, source: string): string; +export declare function waitHeldSubject(branch: string, revision: string): string; export declare const waitHeldSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export declare const waitEndedStream: "SEAT_mesh-pipeline-runner_wait-ended"; -export declare function waitEndedSubject(branch: string, source: string): string; +export declare function waitEndedSubject(branch: string, revision: string): string; export declare const waitEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -94,11 +94,11 @@ export declare const states: { readonly stepOutputs: "step-outputs"; }; -/** The key of one value of the state pipelines under the form ".". */ -export declare function pipelinesByBranchSourceKey(branch: string, source: string): string; +/** The key of one value of the state pipelines under the form ".". */ +export declare function pipelinesByBranchRevisionKey(branch: string, revision: string): string; -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export declare function parsePipelinesByBranchSourceKey(_key: string): { branch: string; source: string } | undefined; +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export declare function parsePipelinesByBranchRevisionKey(_key: string): { branch: string; revision: string } | undefined; /** The key of one value of the state pipelines under the form "..published..". */ export declare function pipelinesByBranchSourcePublishedPackageVersionKey(branch: string, source: string, packagePart: string, version: string): string; @@ -261,7 +261,7 @@ export interface Pipeline { /** the modules the pipeline moves, its take-in's entries (design 47 H §746) */ "movers": Mover[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ + /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -300,7 +300,7 @@ export interface PipelineEnded { /** per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) */ "movers": MoverEnded[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -322,7 +322,7 @@ export interface PipelineStarted { "call"?: string; /** the pipeline's movers */ "movers": Mover[]; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -383,7 +383,7 @@ export interface StepEvent { /** the module's build-input fingerprint at the commit */ "fingerprint"?: string; "module": string; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; "result": StepResult; /** the catalog graph revision of the branch the pipeline runs for */ @@ -490,7 +490,7 @@ export interface ToolchainUsed { "version": string; } -/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72) */ +/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221) */ export interface Wait { "artifact"?: string; /** the step's own ask, where it has one (design 47 G §731) */ @@ -507,7 +507,7 @@ export interface Wait { "module"?: string; /** what the step waits on (design 47 *Health and going back*) */ "on": "person" | "build node" | "revision view" | "pipelines-at-once"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -527,7 +527,7 @@ export interface Wait { /** a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72) */ export interface Waiting { - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; } @@ -800,8 +800,8 @@ export type FullEvent = Fill; /** is the body of the event room. */ export type RoomEvent = Fill; -/** is one value of the state pipelines under the key .. */ -export type PipelinesByBranchSourceValue = Pipeline; +/** is one value of the state pipelines under the key .. */ +export type PipelinesByBranchRevisionValue = Pipeline; /** is one value of the state pipelines under the key ..published... */ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js index 213a3df33..d8ff25e62 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js @@ -40,16 +40,16 @@ export const family = { name: "mesh-pipeline-runner.events", maxAgeDays: 7 }; export const pipelineStartedStream = "SEAT_mesh-pipeline-runner_pipeline-started"; -export function pipelineStartedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source; +export function pipelineStartedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision; } export const pipelineStartedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export const pipelineEndedStream = "SEAT_mesh-pipeline-runner_pipeline-ended"; -export function pipelineEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source; +export function pipelineEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision; } export const pipelineEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -80,16 +80,16 @@ export const stepEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.step-en export const waitHeldStream = "SEAT_mesh-pipeline-runner_wait-held"; -export function waitHeldSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source; +export function waitHeldSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision; } export const waitHeldSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export const waitEndedStream = "SEAT_mesh-pipeline-runner_wait-ended"; -export function waitEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source; +export function waitEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision; } export const waitEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -108,23 +108,23 @@ export const states = { stepOutputs: "step-outputs", }; -/** The key of one value of the state pipelines under the form ".". */ -export function pipelinesByBranchSourceKey(branch, source) { - return branch + "." + source; +/** The key of one value of the state pipelines under the form ".". */ +export function pipelinesByBranchRevisionKey(branch, revision) { + return branch + "." + revision; } -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export function parsePipelinesByBranchSourceKey(_key) { +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export function parsePipelinesByBranchRevisionKey(_key) { let _rest = _key; const _at0 = _rest.indexOf("."); if (_at0 <= 0) return undefined; const branch = _rest.slice(0, _at0); _rest = _rest.slice(_at0 + 1); if (_rest === "") return undefined; - const source = _rest; + const revision = _rest; _rest = ""; if (_rest !== "") return undefined; - return { branch, source }; + return { branch, revision }; } /** The key of one value of the state pipelines under the form "..published..". */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts index 0b53c8714..d4ef255a5 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts @@ -50,7 +50,7 @@ export declare const wantedSubjectAll: "mesh.seat.mesh-wants.event.wanted.*.*"; export declare const pipelineWantedStream: "SEAT_mesh-wants_pipeline-wanted"; -export declare function pipelineWantedSubject(branch: string, source: string): string; +export declare function pipelineWantedSubject(branch: string, revision: string): string; export declare const pipelineWantedSubjectAll: "mesh.seat.mesh-wants.event.pipeline-wanted.*.*"; @@ -412,8 +412,10 @@ export interface PipelineWanted { /** for tier-late, the late tier's movers not running and the machines they do not run on */ "not-running"?: NotRunning[]; "outcome": "running" | "tier-late"; - /** the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746) */ + /** the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4)) */ "pipeline": string; + /** the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event */ + "revision"?: number; "source": number; "v": number; } diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js index a544df9aa..b0c21d4bc 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js @@ -52,8 +52,8 @@ export const wantedSubjectAll = "mesh.seat.mesh-wants.event.wanted.*.*"; export const pipelineWantedStream = "SEAT_mesh-wants_pipeline-wanted"; -export function pipelineWantedSubject(branch, source) { - return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source; +export function pipelineWantedSubject(branch, revision) { + return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision; } export const pipelineWantedSubjectAll = "mesh.seat.mesh-wants.event.pipeline-wanted.*.*"; diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go new file mode 100644 index 000000000..8eb969dd4 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "fmt" + "strconv" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-indexer/internal/index" +) + +// entryAnswer is what indexer_entry answers: the trunk revision read, the entry's packages (the pins: each other-repository +// package with the version it builds against, empty while none is published), its named paths by artifact, its fingerprint +// and its manifest's digest. +type entryAnswer struct { + Module string `json:"module"` + Revision int64 `json:"revision"` + Packages map[string]string `json:"packages"` + Paths map[string][]string `json:"paths"` + Fp string `json:"fp,omitempty"` + ManifestDigest string `json:"manifest-digest,omitempty"` +} + +// entryReadTool is indexer_entry (H-A-204; tools-are-how-a-module-is-spoken-to) over the indexer's catalog-revisions objects: +// one module's entry pins and paths at a trunk revision, small, so a pin can be read without the revision whole (1.3 MB, +// more than the bus carries). +func entryReadTool(o index.Objects) stdio.Tool { + return stdio.Tool{Name: index.EntryPinsTool, + Description: "Read one module's catalog graph entry pins: the packages it builds against with the version each is pinned to " + + "(an empty version while none is published), the paths each artifact is built from, its fingerprint and its manifest " + + "digest, at the newest trunk revision or at a trunk revision given; a small answer, for when the revision whole is too " + + "large for the bus (H-A-204).", + Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module whose entry is read"}, + "revision": map[string]any{"type": "integer", "description": "the trunk revision; none reads the newest"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + module = strings.TrimSpace(module) + if module == "" { + return nil, fmt.Errorf("%s names the module", index.EntryPinsTool) + } + revision, err := revisionArg(args["revision"]) + if err != nil { + return nil, err + } + rev, e, err := index.TrunkEntryAt(context.Background(), o, module, revision) + if err != nil { + return nil, err + } + out := entryAnswer{Module: e.Module, Revision: rev.Revision, Packages: e.Packages, Paths: map[string][]string{}, + Fp: e.Fp, ManifestDigest: e.ManifestDigest} + if out.Packages == nil { + out.Packages = map[string]string{} + } + if e.Inputs != nil && e.Inputs.Paths != nil { + out.Paths = e.Inputs.Paths + } + return out, nil + }} +} + +// revisionArg is the revision argument as a number: absent is 0 (the newest); a number or its digits as a string. +func revisionArg(v any) (int64, error) { + switch r := v.(type) { + case nil: + return 0, nil + case float64: + if r < 0 || r != float64(int64(r)) { + return 0, fmt.Errorf("revision %v is not a trunk revision's number", r) + } + return int64(r), nil + case string: + if strings.TrimSpace(r) == "" { + return 0, nil + } + n, err := strconv.ParseInt(strings.TrimSpace(r), 10, 64) + if err != nil || n < 0 { + return 0, fmt.Errorf("revision %q is not a trunk revision's number", r) + } + return n, nil + } + return 0, fmt.Errorf("revision %v is not a trunk revision's number", v) +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go new file mode 100644 index 000000000..aefb9942a --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + + "mesh-delivery-indexer/internal/index" +) + +// memObjects is catalog-revisions in memory. +type memObjects map[string][]byte + +func (m memObjects) Put(_ context.Context, name string, data []byte) error { + m[name] = data + return nil +} +func (m memObjects) Get(_ context.Context, name string) ([]byte, error) { + d, ok := m[name] + if !ok { + return nil, index.ErrNoObject + } + return d, nil +} +func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil } +func (m memObjects) Names(context.Context) ([]string, error) { + var out []string + for n := range m { + out = append(out, n) + } + return out, nil +} +func (memObjects) Fill(context.Context) (int64, int64, error) { return 0, 0, nil } + +// TestEntryTool: indexer_entry answers one entry's pins, paths, fingerprint and manifest digest, small, at the newest trunk +// revision and at a given one read from its kept object; an unknown module or revision is an error (H-A-204). +func TestEntryTool(t *testing.T) { + o := memObjects{} + keep := func(revision int64, entries ...meshcatalog.CatalogEntry) { + rev := meshcatalog.CatalogRevision{Branch: index.TrunkBranch, Revision: revision} + for _, e := range entries { + e.Manifest = map[string]any{"module": e.Module, "big": strings.Repeat("x", 64<<10)} // the manifest whole is never answered + data, _ := json.Marshal(e) + digest := e.Module + "-" + e.Fp + o[index.EntryObject(digest)] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: digest}) + } + data, _ := json.Marshal(rev) + o[index.BranchObject(index.TrunkBranch, revision)] = data + } + paths := &meshcatalog.EntryInputs{Paths: map[string][]string{"mesh-grants:program": {"modules/mesh-grants", "sdk"}}} + keep(7, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:old", ManifestDigest: "sha256:a", Inputs: paths, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}}, meshcatalog.CatalogEntry{Module: "other", Fp: "src2:o"}) + keep(9, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:new", ManifestDigest: "sha256:b", Inputs: paths, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}}) + tool := entryReadTool(o) + if tool.Name != "indexer_entry" { + t.Fatalf("name %s", tool.Name) + } + for _, c := range []struct { + name string + args map[string]any + want entryAnswer + err string + }{ + {"newest", map[string]any{"module": "mesh-grants"}, entryAnswer{Module: "mesh-grants", Revision: 9, Fp: "src2:new", ManifestDigest: "sha256:b", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}, Paths: paths.Paths}, ""}, + {"given", map[string]any{"module": "mesh-grants", "revision": float64(7)}, entryAnswer{Module: "mesh-grants", Revision: 7, Fp: "src2:old", ManifestDigest: "sha256:a", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}, Paths: paths.Paths}, ""}, + {"given as digits, no paths", map[string]any{"module": "other", "revision": "7"}, entryAnswer{Module: "other", Revision: 7, Fp: "src2:o", + Packages: map[string]string{}, Paths: map[string][]string{}}, ""}, + {"not in the newest", map[string]any{"module": "other"}, entryAnswer{}, "holds no module other"}, + {"revision not kept", map[string]any{"module": "mesh-grants", "revision": float64(8)}, entryAnswer{}, "revision 8 is not kept"}, + {"no module", map[string]any{}, entryAnswer{}, "names the module"}, + } { + t.Run(c.name, func(t *testing.T) { + got, err := tool.Run(c.args) + if c.err != "" { + if err == nil || !strings.Contains(err.Error(), c.err) { + t.Fatalf("error %v, want %q", err, c.err) + } + return + } + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, c.want) { + t.Fatalf("answer %+v, want %+v", got, c.want) + } + data, _ := json.Marshal(got) + if len(data) >= 8<<10 { + t.Fatalf("answer of %d bytes, want under 8 KB", len(data)) + } + }) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index c4311fa4f..a2eba94a8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -193,7 +193,7 @@ func main() { tools = append(tools, catalogTools(trunk.State, kept, watched)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). - tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), + tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), entryReadTool(revisions), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects)) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index bcf468135..76a24231a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -433,7 +433,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro done() done = step("following the assignments") followAssignments(ctx, reader, seatHolders.holders(), process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) { - if err := ix.Recompute(ctx); err != nil { + if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil { logger.Printf("writing the modules again after the assignments of a build module changed: %v", err) } }, logger.Printf) @@ -449,7 +449,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // the toolchain. return ix.ToolchainReported(ctx) }, - recompute: ix.Recompute}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger) + recompute: func(ctx context.Context) error { return ix.RecomputeKept(ctx, "the toolchains read") }}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger) if err := tools.start(); err != nil { return err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index c57388a27..cc42bf67f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -90,7 +90,7 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 }{ {name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"}, {name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"}, - {name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1}, + {name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.13","branch":"main","source":12,"revision":13,"outcome":"tier-late"}`, wanted: true, acked: 1}, {name: "not json", body: `nope`, termd: 1}, {name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"}, {name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"}, @@ -100,8 +100,8 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 switch { case row.wanted: heardPipelineWanted(context.Background(), ix, m, t.Logf) - if ix.wanted != "main.12 tier-late" { - t.Errorf("%s: was handed %q, want main.12 tier-late", row.name, ix.wanted) + if ix.wanted != "main.12.13 tier-late" { // its revision handed beside its source (H-A-214) + t.Errorf("%s: was handed %q, want main.12.13 tier-late", row.name, ix.wanted) } case row.ended: heardPipelineEnded(context.Background(), ix, m, t.Logf) @@ -169,8 +169,11 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte return nil } -func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64, outcome string) error { +func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source, revision int64, outcome string) error { f.wanted = fmt.Sprintf("%s.%d %s", branch, source, outcome) + if revision > 0 { + f.wanted = fmt.Sprintf("%s.%d.%d %s", branch, source, revision, outcome) + } return f.err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go index fdbf67aaa..1c0958584 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go @@ -21,7 +21,7 @@ type runTaker interface { StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error - HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error + HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error JudgePins(ctx context.Context) } @@ -99,7 +99,7 @@ func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf fun _ = m.Term() return } - err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Outcome) + err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Revision, e.Outcome) settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf) if err == nil { ix.JudgePins(ctx) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go index ff1b77f2d..bdfc2a5d5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go @@ -22,8 +22,9 @@ var ( ) // heardVersion reads one message of the consumer as a version, false for any other subject. A package's is kept in its -// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives -// it again. +// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives it +// again. It recomputes no module: a fingerprint moves only with a kept catalog revision, a release's by the re-pin its +// package-published starts, one revision per module release (design 47 H-A-211, H-A-213 (2)). func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) bool { var h index.HeardVersion switch seat, ok := publishedSeats[m.Subject()]; { @@ -51,12 +52,6 @@ func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf if kept { logf("packages: %s %s kept in its series (published %v)", h.Package, h.Version, h.Published) } - if kept && h.Published { - // A series' newest moved: its users' fingerprints take it (DLV-I6), in the same turn as the publish is heard. - if err := ix.Recompute(ctx); err != nil { - logf("writing the modules again after %s %s was published: %v", h.Package, h.Version, err) - } - } _ = m.Ack() return true } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go new file mode 100644 index 000000000..c58fb27fc --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" + + "mesh-delivery-indexer/internal/index" +) + +// ackedMsg is a message that counts its acknowledgements. +type ackedMsg struct { + jetstream.Msg + subject string + data []byte + acked int +} + +func (a *ackedMsg) Subject() string { return a.subject } +func (a *ackedMsg) Data() []byte { return a.data } +func (a *ackedMsg) Ack() error { a.acked++; return nil } + +// unlistedStore is a store whose listing fails: a recompute over it fails. +type unlistedStore struct{ memStore } + +func (unlistedStore) Keys(context.Context) ([]string, error) { + return nil, errors.New("the bus does not answer") +} + +// TestHeardVersionMovesNoFingerprint: a registry's published is kept in its series and acknowledged, and recomputes no +// module: a fingerprint moves only with a kept revision, a release's by its re-pin on the package-published (design 47 +// H-A-211, H-A-213 (2); the pin specification's leftover L3). The modules state here fails any recompute. +func TestHeardVersionMovesNoFingerprint(t *testing.T) { + data, err := json.Marshal(gopackageregistry.PublishedEvent{Kind: "package", Name: "git.example/novox/lib", Version: "0.1.4"}) + if err != nil { + t.Fatal(err) + } + ix := &index.Indexer{Packages: &memStore{vals: map[string][]byte{}}, Modules: &unlistedStore{memStore{vals: map[string][]byte{}}}, + Manifests: &memStore{vals: map[string][]byte{}}} + m := &ackedMsg{subject: gopackageregistry.SubjectPublished, data: data} + if !heardVersion(context.Background(), ix, m, t.Logf) { + t.Fatal("the registry's published is not taken as a version") + } + if m.acked != 1 { + t.Errorf("acknowledged %d times, want once: a publish recomputes nothing that could fail", m.acked) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index cde7d7596..4a42bcdde 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -286,6 +286,14 @@ func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn { return r } +// TakeInAgain takes a repository in again at its kept merge commit (indexer_take_in_again, H-A-208, H-A-210), one take-in +// again at a time beside the held ones. +func (a actor) TakeInAgain(ctx context.Context, repository string) index.TakenIn { + a.s.again.Lock() + defer a.s.again.Unlock() + return a.Indexer.TakeInAgain(ctx, repository) +} + // HeldSkip ends a branch's held source revision unwritten (indexer_take_in_skip): take-in.taken says it skipped. func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn { r := index.TakenIn{Module: branch} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 14625a362..31c7f4a84 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -24,6 +24,8 @@ type takerIn interface { // HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744). HeldAgain(ctx context.Context, branch string) index.TakenIn HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn + // TakeInAgain takes a repository in again at its kept merge commit (H-A-208, H-A-210). + TakeInAgain(ctx context.Context, repository string) index.TakenIn } // takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and @@ -53,6 +55,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.Held != "" { asked, r = "take-in again of a held take-in", ix.HeldAgain(ctx, a.Held) a.Module = a.Held + } else if a.Again != "" { + asked, r = "take-in again of a repository", ix.TakeInAgain(ctx, a.Again) + a.Module = a.Again } else if a.Registry != nil { asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a) } else if a.Recompute { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index c42a2a377..3055a5ac7 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "slices" "strconv" @@ -447,37 +448,42 @@ func TestIndexerWhole(t *testing.T) { // The first revision: no mover, no pin; its entries are the base. sayMoved(t, ctx, js, "main", 10, c10, true) - if first := heard.next(t, "main", 10); len(first.Entries) != 0 || first.Revision != 10 { - t.Fatalf("the first revision said %+v, want revision 10 with no entries", first) + // A revision's number is the indexer's own count, the branch's newest + 1 (design 47 H-A-213 (1)): the later take-ins of + // sources 11 to 13 are counted on from the number of source 10's. + first := heard.next(t, "main", 10) + r := func(source int64) int64 { return first.Revision + source - 10 } + rn := func(source int64) string { return fmt.Sprintf("main.%d", r(source)) } + if len(first.Entries) != 0 || first.Revision != 2 || first.Source != 10 { + t.Fatalf("the first revision said %+v, want revision 2 (the bed's seed is main.1, of source 0) of source 10 with no entries", first) } - untilKey(t, ctx, pins, index.PinKey("main", 10), absent) + untilKey(t, ctx, pins, index.PinKey("main", r(10)), absent) // One take-in, one revision carrying its movers, pinned for its pipeline; said again from the kept movers. sayMoved(t, ctx, js, "main", 11, c11, false) taken := heard.next(t, "main", 11) if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Commit != c11 || taken.Entries[0].Fingerprint == "" || - taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != 11 || taken.Call == "" { + taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != r(11) || taken.Call == "" { t.Fatalf("take-in.taken of main 11 = %+v, want avahi at %s as its one mover", taken, c11) } - rev, _, err := index.ReadRevision(ctx, objects, "main.11") + rev, _, err := index.ReadRevision(ctx, objects, rn(11)) if err != nil || len(rev.Movers) != 1 || rev.Movers[0] != taken.Entries[0] { t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err) } - untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11")) - untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147 + untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(11))) + untilKey(t, ctx, pins, index.PinKey("main", r(10)), pinned("pipeline."+rn(11))) // validate's base, H-A-147 info, err := objects.open(ctx) if err != nil { t.Fatal(err) } - before, err := info.GetInfo(ctx, "main.11") + before, err := info.GetInfo(ctx, rn(11)) if err != nil { t.Fatal(err) } sayMoved(t, ctx, js, "main", 11, c11, false) - if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != 11 { + if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != r(11) { t.Errorf("the source-moved given again said %+v, want the kept movers %+v", again, taken.Entries) } - if after, err := info.GetInfo(ctx, "main.11"); err != nil || after.NUID != before.NUID { + if after, err := info.GetInfo(ctx, rn(11)); err != nil || after.NUID != before.NUID { t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err) } @@ -485,11 +491,11 @@ func TestIndexerWhole(t *testing.T) { // (H-A-147), which the take-in pinned for it. sayMoved(t, ctx, js, "main", 12, c12, false) heard.next(t, "main", 12) - if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below + if err := pins.Delete(ctx, index.PinKey("main", r(12))); err != nil { // the take-in's own pin, so pipeline-started alone writes it below t.Fatal(err) } - sayPipeline(t, ctx, js, false, "main", 12) - untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) + sayPipeline(t, ctx, js, false, "main", r(12)) + untilKey(t, ctx, pins, index.PinKey("main", r(12)), pinned("pipeline."+rn(12))) // A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174). manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") if err != nil { @@ -515,7 +521,7 @@ func TestIndexerWhole(t *testing.T) { heard.acked(t, "main") } restart() - for _, name := range []string{"main.10", "main.11"} { + for _, name := range []string{rn(10), rn(11)} { if _, _, err := index.ReadRevision(ctx, objects, name); err != nil { t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err) } @@ -553,27 +559,27 @@ func TestIndexerWhole(t *testing.T) { t.Errorf("the held source-moved's consumer %+v (%v), want nothing unacknowledged", info, err) } setCap(t, ctx, js, 1<<30) // a larger cap alone wakes nothing: no timer takes a held take-in in - sayPipeline(t, ctx, js, true, "main", 11) + sayPipeline(t, ctx, js, true, "main", r(11)) if got := heard.next(t, "main", 13); len(got.Entries) != 1 || got.Entries[0].Commit != c13 { t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13) } untilKey(t, ctx, held, keys.EscapeName("main"), absent) // main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170). - untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi"))) - untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins + untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(12), index.ReleaseRecord("main", "avahi"))) + untilRevisionGone(t, ctx, js, rn(10)) // the end freed both of main.11's pins // main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end), // frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173). - sayPipeline(t, ctx, js, true, "main", 12) - untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool { + sayPipeline(t, ctx, js, true, "main", r(12)) + untilKey(t, ctx, pins, index.PinKey("main", r(12)), func(v []byte) bool { var p index.PinValue return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi")) }) - wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning}) + wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: rn(12), Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning}) if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil { t.Fatal(err) } - untilKey(t, ctx, pins, index.PinKey("main", 11), absent) - untilRevisionGone(t, ctx, js, "main.11") + untilKey(t, ctx, pins, index.PinKey("main", r(11)), absent) + untilRevisionGone(t, ctx, js, rn(11)) if n := len(heard.pending); n != 0 { t.Errorf("take-in.pending said %d more times for one held source revision, want once", n) } diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_values.go b/modules/mesh-delivery-indexer/internal/index/candidate_values.go index 62cc8b33f..f443f807e 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidate_values.go +++ b/modules/mesh-delivery-indexer/internal/index/candidate_values.go @@ -39,7 +39,9 @@ func checkCandidateModule(m meshcatalog.CandidatesByGroupValuePullsItemFpsValue) return fmt.Errorf("index: the named path %q names no git object", p) } } - if err := sdkstep.PackageVersions(m.Packages).Check(); err != nil { + // An entry names a package with no published version yet with an empty version (design 47 H S8, H-A-204): it is checked + // as an entry's packages, not as an ask's. + if err := sdkstep.PackageVersions(m.Packages).CheckEntry(); err != nil { return err } if m.Fp2 != "" { diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_values_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_values_test.go new file mode 100644 index 000000000..fa25038af --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/candidate_values_test.go @@ -0,0 +1,30 @@ +package index + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" +) + +// TestCheckCandidateModulePins: an entry's package with no published version (an empty pin) passes a run on request's +// candidate; a pin that is no version is refused (design 47 H S8, the pin specification's item 4). +func TestCheckCandidateModulePins(t *testing.T) { + for _, c := range []struct { + name string + packages map[string]string + ok bool + }{ + {"an empty pin", map[string]string{"git.example/novox/lib/go": ""}, true}, + {"a version", map[string]string{"git.example/novox/lib/go": "v0.1.4"}, true}, + {"no version", map[string]string{"git.example/novox/lib/go": "latest"}, false}, + } { + t.Run(c.name, func(t *testing.T) { + m := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Fp2: "src2:" + strings.Repeat("ab", 32), Packages: c.packages} + err := checkCandidateModule(m) + if (err == nil) != c.ok { + t.Errorf("checkCandidateModule(%v) = %v, want ok %v", c.packages, err, c.ok) + } + }) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go index 5e2399312..66319e487 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -36,13 +36,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { for _, n := range names { have[n] = true } - for _, want := range []string{"main.5", "feature/x=2ey.7", "feature/x=2ey.8"} { + for _, want := range []string{"main.1", "feature/x=2ey.1", "feature/x=2ey.2"} { if !have[want] { t.Errorf("the objects hold %v, want %s", names, want) } } - if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 8 { - t.Errorf("said %+v, want main 5, then the escaped branch 7 and 8", said) + if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 || said[2].Source != 8 { + t.Errorf("said %+v, want main 1 of source 5, then the escaped branch 1 and 2 of sources 7 and 8", said) } answer := func(branch string, revision int64) meshcatalog.RevisionAnswer { a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Revision: revision}) @@ -51,14 +51,14 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { } return a } - if a := answer("main", 0); a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { + if a := answer("main", 0); a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main answered %+v", a) } - if a := answer("feature/x=2ey", 0); a.Revision.Revision != 8 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { - t.Errorf("the branch's newest answered %+v, want its revision 8 at c3", a.Revision) + if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { + t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision) } - if a := answer("feature/x=2ey", 7); a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's revision 7 answered %+v", a.Revision) + if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's revision 1 answered %+v", a.Revision) } if a := answer("beta", 0); !a.NoneYet { t.Errorf("a branch with no revision answered %+v, want none-yet", a) @@ -68,15 +68,22 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) { return Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Source: source}) } - if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { + if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's source 7 answered %+v (%v), want its revision 7 at c2", a.Revision, err) + if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's source 7 answered %+v (%v), want its revision 1 at c2", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 8 { + if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 { t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err) } + // H-A-213: a re-pin's revision of source 8 answers below the take-in's: source 8 answers revision 2. + if _, err := ix.keepBranchEntries(ctx, "feature/x.y", 8, releaseCall("mesh-sdk", "0.1.27"), tip("c4"), nil); err != nil { + t.Fatal(err) + } + if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 || a.Entries[0].Commit != "c3" { + t.Errorf("the branch's source 8 after a re-pin answered %+v (%v), want the take-in's revision 2", a.Revision, err) + } if _, err := bySource("feature/x=2ey", 9); err == nil { t.Error("a source no revision was taken in from was answered") } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ec1e77172..38a96ead6 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "slices" "strconv" "strings" @@ -87,6 +88,37 @@ func BranchRevisionOf(branch, name string) (int64, bool) { return r, err == nil && r > 0 } +// watchedBranches are the watched branches the bucket keeps a revision of, the first included when it keeps one, sorted: +// each name . but an entry's, a candidate's and the earlier forms' (earlierForm). +func (ix *Indexer) watchedBranches(ctx context.Context) ([]string, error) { + if ix.Revisions == nil { + return nil, nil + } + names, err := ix.Revisions.Names(ctx) + if err != nil { + return nil, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + } + seen := map[string]bool{} + for _, n := range names { + if strings.HasPrefix(n, "entry.") || strings.HasPrefix(n, "candidate.") || earlierForm(n) { + continue + } + i := strings.LastIndexByte(n, '.') + if i <= 0 { + continue + } + if r, err := strconv.ParseInt(n[i+1:], 10, 64); err != nil || r <= 0 { + continue + } + branch, err := keys.UnescapeName(n[:i]) + if err != nil { + continue + } + seen[branch] = true + } + return slices.Sorted(maps.Keys(seen)), nil +} + // TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one. func TrunkRevisionOf(name string) (int64, bool) { return BranchRevisionOf(TrunkBranch, name) } @@ -210,6 +242,54 @@ func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Ca } } +// TrunkEntryAt is a module's entry in a trunk revision, with that revision's list (H-A-204): for revision 0 the newest trunk +// revision read as entryOf reads it (readNewestOf); else the kept list of that trunk revision and only the module's entry +// object, so one entry is read without the revision whole. +func TrunkEntryAt(ctx context.Context, o Objects, module string, revision int64) (meshcatalog.CatalogRevision, meshcatalog.CatalogEntry, error) { + if revision == 0 { + rev, entries, found, err := readNewestOf(ctx, o, TrunkBranch) + if err != nil { + return rev, meshcatalog.CatalogEntry{}, err + } + if !found { + return rev, meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet") + } + for _, e := range entries { + if e.Module == module { + return rev, e, nil + } + } + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", rev.Revision, module) + } + name := BranchObject(TrunkBranch, revision) + var rev meshcatalog.CatalogRevision + data, err := o.Get(ctx, name) + if errors.Is(err, ErrNoObject) { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d is not kept", revision) + } + if err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err) + } + if err := json.Unmarshal(data, &rev); err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err) + } + for _, ref := range rev.Entries { + if ref.Module != module { + continue + } + data, err := o.Get(ctx, EntryObject(ref.Digest)) + if err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, module, ref.Digest, err) + } + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(data, &e); err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s: %w", name, module, err) + } + return rev, e, nil + } + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", revision, module) +} + // changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name. func changedEntries(before, after []meshcatalog.EntryRef) []string { was := map[string]string{} @@ -266,17 +346,22 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata } // KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design -// 47 H §743, H-A-32): ., the trunk's included, numbered by the source, and says catalog-revision naming the -// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline, -// pipeline.., in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written -// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it. +// 47 H §743, H-A-32), as keepBranchEntries does with no call. func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { - return ix.keepBranchEntries(ctx, branch, src, entries, movers) + _, err := ix.keepBranchEntries(ctx, branch, src, revisionCall{}, entries, movers) + return err } -func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { +// keepBranchEntries keeps entries as the next catalog revision of a watched branch, ., the trunk's included: n is +// the indexer's own count, the branch's newest revision + 1, and src the gate's source revision it was read at, its field +// source (design 47 H-A-213 (1)); call is what made it (H-A-212, H-A-213 (3)): a take-in's the gate's call, a re-pin's +// "release of ", a take-in again's "take-in again of ". It says catalog-revision naming the +// branch, its source and its call. movers are kept on the revision (H-A-92), and the revision pinned for its one pipeline, +// pipeline.., in the same write when there is one (A-449a, H-A-213 (4)), with the branch's previous revision +// (H-A-147). It answers the number it kept; ErrObjectsFull is a store too full to keep it. +func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, call revisionCall, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) (int64, error) { if ix.Revisions == nil { - return nil + return 0, nil } if branch == "" { branch = TrunkBranch @@ -286,28 +371,24 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int newest, _, err := newestOf(ctx, ix.Revisions, branch) if errors.Is(err, process.ErrNotYet) { ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects) - return err + return 0, err } if err != nil { ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err) - return fmt.Errorf("the newest catalog graph revision: %w", err) - } - if _, err := ix.Revisions.Get(ctx, BranchObject(branch, src)); err == nil { - ix.logf("catalog graph: revision %s is kept already: not written again", BranchObject(branch, src)) - return nil - } else if !errors.Is(err, ErrNoObject) { - return fmt.Errorf("revision %s: %w", BranchObject(branch, src), err) + return 0, fmt.Errorf("the newest catalog graph revision: %w", err) } + number := newest.Revision + 1 entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208) at := ix.now().UTC() - rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: src, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: number, Kind: "trunk", Branch: branch, Source: src, Call: call.Call, + CallKind: call.Kind, Module: call.Module, Version: call.Version, Repository: call.Repository, At: &at}, entries) if err != nil { - return err + return 0, err } rev.Movers = movers // The base of the validate step's case before: the newest revision kept before this write, which the take-in pins for its // pipeline below, named on the revision in this same write (H-A-147, H-A-151, H-A-154); none on the branch's first take-in. - pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 && newest.Revision != src + pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 if pinsBase { rev.Base = newest.Revision } @@ -319,32 +400,32 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull) ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity) ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused}) - return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull) + return 0, fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull) } if err := ix.indexEntries(ctx, rev, entries); err != nil { ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) - return err + return 0, err } if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) - return err + return 0, err } // The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a). if len(movers) > 0 && ix.Pins != nil { - record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10))) - if err := ix.pin(ctx, PinKey(branch, src), record); err != nil { - ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err) - return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err) + record := PipelineRecord(runner.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(number, 10))) + if err := ix.pin(ctx, PinKey(branch, number), record); err != nil { + ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, number), record, err) + return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, number), err) } // The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of // the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it. if pinsBase { if err := ix.pin(ctx, PinKey(branch, rev.Base), record); err != nil { ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err) - return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err) + return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err) } } else { - ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record) + ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, number), record) } } // The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it. @@ -364,16 +445,48 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 { ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity}) } - ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed)) + ix.logf("catalog graph: revision %d of source revision %d kept (%s), %d entries, %d changed", rev.Revision, src, call.Call, len(rev.Entries), len(changed)) if ix.SayCatalogRevision != nil { if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind, - Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { + Branch: keys.EscapeName(branch), Source: src, Call: call.Call, CallKind: call.Kind, Module: call.Module, Version: call.Version, + Repository: call.Repository, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err) } } - return nil + return number, nil } +// revisionCall is what made a catalog revision (design 47 H-A-213 (3), H-A-215): call, the words for a person; Kind, one of the +// seat mesh-catalog's three call kinds, which a reader reads instead of the words; Module and Version for a release, Repository +// for a take-in again. The zero value names no call (a revision kept by hand or a test). +type revisionCall struct { + Call, Kind, Module, Version, Repository string +} + +// takeInCall is a take-in's: the gate's call its source revision names. +func takeInCall(call string) revisionCall { + return revisionCall{Call: call, Kind: meshcatalog.CatalogRevisionCallKindTakeIn} +} + +// releaseCall is a re-pin's on the release of module at version (H-A-213 (2)). +func releaseCall(module, version string) revisionCall { + return revisionCall{Call: ReleaseCall(module, version), Kind: meshcatalog.CatalogRevisionCallKindRelease, Module: module, Version: version} +} + +// takeInAgainCall is a take-in again's of repository (H-A-212). +func takeInAgainCall(repository string) revisionCall { + return revisionCall{Call: TakeInAgainCall(repository), Kind: meshcatalog.CatalogRevisionCallKindTakeInAgain, Repository: repository} +} + +// recomputeCall is a recompute's that moved a fingerprint, its call naming the reason (H-A-217). +func recomputeCall(reason string) revisionCall { + return revisionCall{Call: RecomputeCall(reason), Kind: meshcatalog.CatalogRevisionCallKindRecompute} +} + +// RecomputeCall is the call of the catalog revision a recompute keeps: its reason, such as the assignments of a build module, +// a toolchain or a hand act (design 47 H-A-217). +func RecomputeCall(reason string) string { return "recompute on " + reason } + // HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held). const HeldObjectsFull = "objects full" diff --git a/modules/mesh-delivery-indexer/internal/index/entry_verb.go b/modules/mesh-delivery-indexer/internal/index/entry_verb.go index 96aee4e8c..e310ac844 100644 --- a/modules/mesh-delivery-indexer/internal/index/entry_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/entry_verb.go @@ -21,6 +21,8 @@ const ( EntryIndexState = "entry-index" // EntryIndexTool is the tool that reads a fingerprint's key and its object's digest (class read). EntryIndexTool = "indexer_entries" + // EntryPinsTool is the tool that reads one module's entry pins and paths at a trunk revision (class read; H-A-204). + EntryPinsTool = "indexer_entry" ) // EntryIndexValue is one key of entry-index: the digest of the entry object of the fingerprint the key names. diff --git a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go index 5de12ff77..de25475fa 100644 --- a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go +++ b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go @@ -49,7 +49,7 @@ func TestAKeptRevisionIndexesItsEntries(t *testing.T) { ctx := context.Background() fp := "src2:" + strings.Repeat("d", 64) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }} - if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" { diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index c947f68e1..5e8774ad8 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -260,6 +260,32 @@ func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error) } return a.Result, nil }}, + {Name: TakeInAgainTool, + Description: "Take a repository in again now at the merge commit the indexer keeps of it (design 47 H-A-208, H-A-210): its " + + "module graph is read again (each module's requirements, transitively) and the modules state recomputed as a " + + "package's publish re-pins it, so a changed take-in rule reaches a repository with no merge since; pins and " + + "fingerprints move, no catalog revision is kept and the gate is not asked. An unknown repository is refused.", + Input: map[string]any{"type": "object", "required": []string{"repository"}, "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "the repository as owner/name, such as novox/mesh-host"}, + "why": map[string]any{"type": "string", "description": "why it is taken in again now"}}}, + Run: func(args map[string]any) (any, error) { + repository := str(args, "repository") + if repository == "" { + return nil, errors.New("name the repository (repository) as owner/name") + } + data, err := json.Marshal(TakeInAsk{V: 1, Again: repository, Why: str(args, "why"), By: stdio.CallerOf(args), At: now().UTC()}) + if err != nil { + return nil, err + } + a, err := ask("again."+keys.EscapeName(repository), data) + if err != nil { + return nil, err + } + if a.Result != nil && !a.Result.Stored { + return a.Result, fmt.Errorf("%s is not taken in again: %s", repository, a.Result.Answer) + } + return a.Result, nil + }}, {Name: TakeInSkipTool, Description: "End a watched branch's held source-moved unwritten (design 47 H §744): it is acknowledged, take-in.taken is said " + "with no entries and skipped, and the hand act is recorded through mesh-hand-acts before it is asked. indexer_pending " + diff --git a/modules/mesh-delivery-indexer/internal/index/package_trees_test.go b/modules/mesh-delivery-indexer/internal/index/package_trees_test.go index 999bb2205..06c1fc063 100644 --- a/modules/mesh-delivery-indexer/internal/index/package_trees_test.go +++ b/modules/mesh-delivery-indexer/internal/index/package_trees_test.go @@ -137,7 +137,7 @@ func TestTheEntryPinsEveryRequirementOfTheModuleGraph(t *testing.T) { t.Errorf("packages = %+v, want the client's requirement and state's mesh-sdk, and no module of the repository", m.Inputs2) } builds := "git.novox.be/novox/mesh-delivery/modules/b/builds" - pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery") + pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery", nil) if _, pinned := pins[builds]; pinned || len(pins) != 1 { t.Errorf("pins = %v, want %s only, never a module of the repository", pins, conditions) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 33816fa80..92a543d29 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -119,8 +119,9 @@ func ReleaseRecord(branch, module string) string { // every revision of the branch below it; a mover the revision holds no release of (superseded, no version released) keeps its // older release's revision. Outcome tier-late frees nothing: its later tiers' wants are not written, though their release // records are held, and the same pipeline's later running frees them. A pipeline whose revision is not kept frees nothing and -// says so; one applied twice changes nothing. -func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error { +// says so; one applied twice changes nothing. The pipeline's revision is the one the event names (design 47 H-A-214); an event +// naming none (revision zero, said before the field) is of the source's lowest-numbered revision, the take-in's. +func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error { if ix.Pins == nil || ix.Revisions == nil { return errors.New("the indexer keeps no state " + PinsState) } @@ -128,12 +129,21 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc ix.logf("pins: pipeline-wanted of %s %d is %s: it frees no release until its wants are written everywhere (H-A-174)", branch, source, outcome) return nil } - name, err := revisionOfSource(ctx, ix.Revisions, branch, source) - if err != nil { - ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err) - return nil + var name string + if revision > 0 { + name = RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: revision}) + } else { + var err error + if name, err = revisionOfSource(ctx, ix.Revisions, branch, source); err != nil { + ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err) + return nil + } } rev, _, err := ReadRevision(ctx, ix.Revisions, name) + if revision > 0 && errors.Is(err, ErrNoObject) { + ix.logf("pins: pipeline-wanted of %s revision %d frees no release: %s is not kept", branch, revision, name) + return nil + } if err != nil { return fmt.Errorf("%s: %w", name, err) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index 4a255ebdf..c6de51474 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -106,42 +106,54 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te t.Fatal(err) } } - end := func(source int64, outcome, version string) { + endOf := func(source, revision int64, outcome, version string) { t.Helper() - e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome, + e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", revision), Revision: revision, Source: source, Outcome: outcome, Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}} if err := ix.HeardPins(ctx, nil, &e); err != nil { t.Fatal(err) } ix.JudgePins(ctx) } + end := func(source int64, outcome, version string) { t.Helper(); endOf(source, source, outcome, version) } + // An event naming no revision (said before the field) is of the source's lowest-numbered revision, the take-in's (H-A-214). wanted := func(source int64, outcome string) { t.Helper() - if err := ix.HeardPipelineWanted(ctx, "main", source, outcome); err != nil { + if err := ix.HeardPipelineWanted(ctx, "main", source, 0, outcome); err != nil { t.Fatal(err) } ix.JudgePins(ctx) } - take(10) - end(10, runner.PipelineEndedOutcomePassed, "0.1.10") - take(11) - end(11, runner.PipelineEndedOutcomeFailed, "") - want("a passed release past a later failed pipeline", "main.10", "main.11") - take(12) - end(12, runner.PipelineEndedOutcomePassed, "") - wanted(12, meshwants.PipelineWantedOutcomeRunning) - want("a passed pipeline releasing no version frees no older release", "main.10", "main.12") - take(13) - end(13, runner.PipelineEndedOutcomePassed, "0.1.13") - want("two releases, the unreleased one between them gone", "main.10", "main.13") - wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing - want("a tier-late pipeline-wanted", "main.10", "main.13") - wanted(13, meshwants.PipelineWantedOutcomeRunning) - wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more - want("the newer release written everywhere", "main.13") - if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { - t.Errorf("the pin of main.13 holds %s, want its release alone", v) + take(4) + end(4, runner.PipelineEndedOutcomePassed, "0.1.10") + take(5) + end(5, runner.PipelineEndedOutcomeFailed, "") + want("a passed release past a later failed pipeline", "main.4", "main.5") + take(6) + end(6, runner.PipelineEndedOutcomePassed, "") + wanted(6, meshwants.PipelineWantedOutcomeRunning) + want("a passed pipeline releasing no version frees no older release", "main.4", "main.6") + take(7) + end(7, runner.PipelineEndedOutcomePassed, "0.1.13") + want("two releases, the unreleased one between them gone", "main.4", "main.7") + wanted(7, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing + want("a tier-late pipeline-wanted", "main.4", "main.7") + wanted(7, meshwants.PipelineWantedOutcomeRunning) + wanted(7, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more + want("the newer release written everywhere", "main.7") + if v, _, _, _ := ix.Pins.Get(ctx, "main.7"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { + t.Errorf("the pin of main.7 holds %s, want its release alone", v) } + // H-A-214: a re-pin's revision 8 of the same source 7 releases 0.1.14; its pipeline-wanted names revision 8, so the release + // on revision 8 frees the older one on revision 7, which the source's lowest-numbered revision would not. + take(7) + endOf(7, 8, runner.PipelineEndedOutcomePassed, "0.1.14") + want("a re-pin's release beside the take-in's", "main.7", "main.8") + if err := ix.HeardPipelineWanted(ctx, "main", 7, 8, meshwants.PipelineWantedOutcomeRunning); err != nil { + t.Fatal(err) + } + ix.JudgePins(ctx) + want("the re-pin's release written everywhere", "main.8") } func mustRecords(t *testing.T, v []byte) []string { @@ -168,21 +180,22 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { before []int64 // sources taken in with no mover before the take-in of source 5 whileRuns []string afterTheEnd []string - base int64 // the base main.5 names (H-A-154) + at int64 // the number the take-in of source 5 keeps: the branch's newest + 1 (H-A-213 (1)) + base int64 // the base that revision names (H-A-154) passed bool // main.5's pipeline passed: its release pins main.5 unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173) then []int64 // sources taken in with no mover after the end released []string }{ - {name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4}, - {name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2}, - {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, - {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, - base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}}, - {name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, - base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}}, - {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, - base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}}, + {name: "the previous revision kept and pinned", before: []int64{3, 4}, at: 3, whileRuns: []string{"main.2", "main.3"}, afterTheEnd: []string{"main.3"}, base: 2}, + {name: "the newest kept before, whatever its source", before: []int64{2}, at: 2, whileRuns: []string{"main.1", "main.2"}, afterTheEnd: []string{"main.2"}, base: 1}, + {name: "no revision before it pins only its own", at: 1, whileRuns: []string{"main.1"}, afterTheEnd: []string{"main.1"}}, + {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"}, + base: 1, passed: true, then: []int64{6}, afterTheEnd: []string{"main.2", "main.3"}, released: []string{"main.2"}}, + {name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"}, + base: 1, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.3"}}, + {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"}, + base: 1, then: []int64{6}, afterTheEnd: []string{"main.3"}}, } { t.Run(tc.name, func(t *testing.T) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), @@ -206,18 +219,19 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { keep(s, nil) } keep(5, mover) - if rev, _, err := ReadRevision(ctx, ix.Revisions, "main.5"); err != nil || rev.Base != tc.base { - t.Errorf("main.5 names the base %d (%v), want %d, the revision its take-in pinned (H-A-154)", rev.Base, err, tc.base) + at := fmt.Sprintf("main.%d", tc.at) + if rev, _, err := ReadRevision(ctx, ix.Revisions, at); err != nil || rev.Base != tc.base || rev.Source != 5 { + t.Errorf("%s names the source %d and the base %d (%v), want 5 and %d, the revision its take-in pinned (H-A-154)", at, rev.Source, rev.Base, err, tc.base) } if got := kept(); !slices.Equal(got, tc.whileRuns) { t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns) } for _, k := range tc.whileRuns { - if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) { + if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline." + at}) { t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v) } } - end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed, + end := runner.PipelineEnded{Branch: "main", Pipeline: at, Revision: tc.at, Outcome: runner.PipelineEndedOutcomeFailed, Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}} if tc.unreleased { end.Movers[0].Version = "" diff --git a/modules/mesh-delivery-indexer/internal/index/repin_test.go b/modules/mesh-delivery-indexer/internal/index/repin_test.go new file mode 100644 index 000000000..584c37083 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/repin_test.go @@ -0,0 +1,100 @@ +package index + +import ( + "context" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Design 47 H-A-204, H-A-201, H-A-165: an entry's packages name every requirement of its module graph of another +// repository: one the form's versions leave out (an import form naming only what the build imported) is pinned by its +// series' newest published version, and one none publishes yet is kept with no version, never left out. +func TestAnEntryNamesEveryRequirementPinnedOrWaiting(t *testing.T) { + const ( + wireguard = "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go" + busgrants = "git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants" + sdk = "git.novox.be/novox/mesh-sdk/go" + ) + series := map[string]SeriesPackage{"wg": {Name: "mesh-wireguard/go", GoModule: wireguard, Series: catalogue.DefaultSeries, + Newest: &meshcatalog.PackagesByPackageSeriesValueNewest{Version: "0.1.47"}}} + pins := releasedPins(sdkstep.PackageVersions{sdk: "c0ffee"}, series, catalogue.Manifest{}, "novox/mesh-catalog", []string{busgrants, sdk, wireguard}) + want := sdkstep.PackageVersions{sdk: "", wireguard: "v0.1.47", busgrants: ""} + if len(pins) != len(want) { + t.Fatalf("pins = %v, want %v", pins, want) + } + for p, v := range want { + if got, named := pins[p]; !named || got != v { + t.Errorf("the pin of %s is %q (named %v), want %q", p, got, named, v) + } + } +} + +// Design 47 H-A-204, H-A-165: a release re-pins every kept entry naming one of its packages, of any repository and +// changed or not, by its pins or by its kept requirements alone: here g, of another repository, holds no pin of mesh-sdk. +// H-A-211, H-A-212, H-A-213: on every watched branch holding g, the re-pin keeps the branch's next revision, of the same +// source as the one before it, with the call "release of mesh-sdk 0.1.27" and g as its mover at its new fingerprint, and +// says catalog-revision with that call. +func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) { + ctx := context.Background() + objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} + ix, mods, _ := ptpIndexer(t, objects) + ix.Revisions, ix.Packages = newMemObjects(1<<20), newMem() + reportToolchain(t, ix, "mesh-build-go", "n1", "go", "1.26.0") + var said []meshcatalog.CatalogRevisionEvent + ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { said = append(said, e); return nil } + f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil) + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + g, _ := moduleOf(t, mods, "g") + if g.Inputs2 == nil || !slices.Contains(g.Inputs2.Packages, "git.novox.be/novox/mesh-sdk/go") { + t.Fatalf("g = %+v, want mesh-sdk among its kept requirements", g.Inputs2) + } + g.Packages = nil + mods.put("g", mustEncode(t, g)) + mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40), + Path: meshcatalogclient.RootPath, None2: None2Pending})) + entries, err := ix.viewEntries(ctx) + if err != nil { + t.Fatal(err) + } + sources := map[string]int64{"main": 5, "feature/x": 9} + for branch, src := range sources { + if err := ix.KeepBranchEntries(ctx, branch, src, entries, nil); err != nil { + t.Fatal(err) + } + } + said = nil + if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.27", Published: true}); err != nil { + t.Fatal(err) + } + if n, err := ix.Repin(ctx, "mesh-sdk", "0.1.27"); err != nil || n != 2 { + t.Errorf("a release of mesh-sdk re-pinned %d modules (%v), want g on each of the two branches, which requires it with no pin", n, err) + } + now, _ := moduleOf(t, mods, "g") + if now.Fp2 == "" || now.Fp2 == g.Fp2 || now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.27" { + t.Fatalf("g after the re-pin = %s pinned %v, want a new fingerprint pinned to v0.1.27 (was %s)", now.Fp2, now.Packages, g.Fp2) + } + for branch, src := range sources { + rev, kept, found, err := readNewestOf(ctx, ix.Revisions, branch) + if err != nil || !found || rev.Revision != 2 || rev.Source != src || rev.Call != "release of mesh-sdk 0.1.27" || + rev.CallKind != meshcatalog.CatalogRevisionCallKindRelease || rev.Module != "mesh-sdk" || rev.Version != "0.1.27" || // H-A-215 + len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != now.Fp2 { + t.Errorf("%s's newest = %+v (%v), want revision 2 of source %d, its call the release, g its mover at %s", branch, rev, err, src, now.Fp2) + continue + } + if i := slices.IndexFunc(kept, func(e meshcatalog.CatalogEntry) bool { return e.Module == "g" }); i < 0 || kept[i].Fp != now.Fp2 { + t.Errorf("%s's newest holds no entry of g at %s", branch, now.Fp2) + } + } + if len(said) != 2 || said[0].Call != "release of mesh-sdk 0.1.27" || said[1].Call != said[0].Call || + said[0].CallKind != meshcatalog.CatalogRevisionEventCallKindRelease || said[1].Module != "mesh-sdk" || said[1].Version != "0.1.27" { + t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index 279e41e81..cd98aba6f 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -272,7 +272,7 @@ func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn r.Answer = "the newest catalog graph revision cannot be read: " + err.Error() return r } - if err := ix.Recompute(ctx); err != nil { + if err := ix.RecomputeKept(ctx, fmt.Sprintf("a hand act by %s (%s)", by, why)); err != nil { r.Answer = "the recompute failed: " + err.Error() return r } diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index 204028833..2667fbb4e 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -167,18 +167,24 @@ func revisionOfSource(ctx context.Context, o Objects, branch string, source int6 } } slices.SortFunc(found, func(a, b int64) int { return int(b - a) }) + // A re-pin or a take-in again keeps a later revision of the same source (H-A-211, H-A-213): the source's own revision is + // the lowest numbered of it, the take-in's. + lowest := "" for _, r := range found { name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: r}) rev, _, err := ReadRevision(ctx, o, name) if err != nil { return "", err } - if rev.Source == source { - return name, nil - } if rev.Source < source { break } + if rev.Source == source { + lowest = name + } + } + if lowest != "" { + return lowest, nil } return "", fmt.Errorf("no revision of %q was taken in from the source revision %d", branch, source) } diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again.go b/modules/mesh-delivery-indexer/internal/index/take_in_again.go new file mode 100644 index 000000000..e27a23f1c --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again.go @@ -0,0 +1,78 @@ +package index + +import ( + "context" + "errors" + "fmt" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// TakeInAgainTool takes a repository in again at its kept merge commit (design 47 H-A-208, H-A-210): a rule change of the +// take-in (such as the walk of the module graph, H-A-201) reaches a repository with no merge since. +const TakeInAgainTool = "indexer_take_in_again" + +// TakeInAgainCall is the call of the catalog revision a take-in again keeps (design 47 H-A-212, H-A-213 (3)). +func TakeInAgainCall(repository string) string { return "take-in again of " + repository } + +// TakeInAgain re-reads the repository's module graph at the merge commit its taken key keeps (wholeAt, the reading of every +// take-in: the kept requirements of each of its modules, H-A-201, H-A-157) and applies it as a take-in does, whose recompute +// of the modules state is the one a publish re-pin runs (recomputeLocked): pins and fingerprints move as they would there. +// It keeps the result as the trunk's next catalog revision of the same source, its call TakeInAgainCall (H-A-212, H-A-213), +// and asks no gate. A repository with no taken key is refused, said in the answer. +func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn { + repository = strings.TrimSpace(repository) + r := TakenIn{Module: repository, Repository: repository} + refuse := func(format string, a ...any) TakenIn { + r.Answer = "refused: " + fmt.Sprintf(format, a...) + ix.logf("take-in again of %s: %s", repository, r.Answer) + return r + } + if ix.Taken == nil || ix.Modules == nil { + return refuse("the indexer keeps no taken merges or no modules, so nothing is taken in again") + } + key, err := trunk.RepositoryKey(repository) + if err != nil { + return refuse("%q names no repository as owner/name", repository) + } + value, _, found, err := ix.Taken.Get(ctx, key) + if errors.Is(err, process.ErrNotYet) { + return refuse("the state taken is not on the bus yet") + } + if err != nil { + return refuse("taken.%s cannot be read: %v", key, err) + } + if !found { + return refuse("the indexer keeps no merge of %s (no key taken.%s): an unknown repository, or one never taken in", repository, key) + } + t, err := takein.DecodeTaken(value) + if err != nil || t.Merge == "" { + return refuse("taken.%s names no merge commit", key) + } + ix.pendingMu.Lock() // one take-in step at a time (pending.go) + defer ix.pendingMu.Unlock() + f, err := ix.wholeAt(ctx, repository, t.Merge) + if err != nil { + return refuse("%v", err) + } + f.Branch = TrunkBranch + applied, err := ix.ApplyMessage(ctx, f) + if err != nil { + return refuse("taking %s at %.12s in again: %v", repository, t.Merge, err) + } + // The take-in again keeps its revision as a re-pin does: the branch's next, of the same source, its call naming the + // repository, its movers every module whose fingerprint moved (design 47 H-A-212, H-A-213). + if err := ix.keepRepinned(ctx, ix, TrunkBranch, takeInAgainCall(repository), nil); err != nil { + return refuse("taken in again at %.12s, but its catalog revision is not kept: %v", t.Merge, err) + } + r.Stored, r.Commit = true, t.Merge + r.Answer = fmt.Sprintf("taken in again at %.12s: %d modules written", t.Merge, len(applied.Written)) + if len(applied.Written) > 0 { + r.Answer += " (" + strings.Join(applied.Written, ", ") + ")" + } + ix.logf("take-in again of %s: %s", repository, r.Answer) + return r +} diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go new file mode 100644 index 000000000..1fe65e2d0 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go @@ -0,0 +1,113 @@ +package index + +import ( + "context" + "maps" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" +) + +// Design 47 H-A-208, H-A-210: a repository taken in before the walk of the module graph (H-A-201) keeps its modules' direct +// requirements only; indexer_take_in_again re-reads it at its kept merge commit and recomputes the modules state, so the +// module's kept requirements and pins gain the transitive package, and keeps the trunk's next catalog revision of the same +// source with the call "take-in again of " (H-A-212, H-A-213). An unknown repository is refused. +func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) { + const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} + ix, mods, _ := ptpIndexer(t, objects) + ix.Taken, ix.Revisions, ix.Packages = newMem(), newMemObjects(1<<20), newMem() + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + direct := map[string]string{"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n"} + walked := map[string]string{"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n"} + goMods := direct // before the rule: the client's go.mod reads as requiring nothing + ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) { + if c, ok := goMods[dir]; ok { + return []byte(c), nil + } + if dir == "modules/c/client" { + return []byte("module git.novox.be/novox/mesh-delivery/modules/c/client\n"), nil + } + return []byte(ptpGoMods[dir]), nil + } + f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, + map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}}) + var readAt []string + ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) { + readAt = append(readAt, repository+"@"+sha) + return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil + } + ctx := context.Background() + for p, v := range map[string]string{conditions: "0.1.4", "git.novox.be/novox/mesh-sdk/go": "0.1.27"} { + if _, err := ix.HearVersion(ctx, HeardVersion{Package: p, GoModule: p, Version: v, Published: true}); err != nil { + t.Fatal(err) + } + } + // the modules that hold the two packages, so their published versions pin them (fingerprint.HolderOf) + mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40), + Path: meshcatalogclient.RootPath, None2: None2Pending})) + mods.put("meshconditionsclient", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-contracts", Commit: strings.Repeat("2", 40), + Path: "health-and-repair/go/clients/meshconditionsclient", None2: None2Pending})) + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + before, _ := moduleOf(t, mods, "g") + if before.Inputs2 == nil || slices.Contains(before.Inputs2.Packages, conditions) || strings.Contains(before.None2, conditions) { + t.Fatalf("before: packages = %+v (%s), want no transitive requirement yet", before.Inputs2, before.None2) + } + entries, err := ix.viewEntries(ctx) + if err != nil { + t.Fatal(err) + } + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 4, entries, nil); err != nil { + t.Fatal(err) + } + if r := ix.TakeInAgain(ctx, "novox/unknown"); r.Stored || !strings.HasPrefix(r.Answer, "refused:") { + t.Errorf("unknown repository: %+v, want refused", r) + } + goMods = maps.Clone(direct) + maps.Copy(goMods, walked) + r := ix.TakeInAgain(ctx, "novox/mesh-delivery") + if !r.Stored || r.Commit != merge { + t.Fatalf("take-in again = %+v, want stored at the kept merge %s", r, merge) + } + if len(readAt) == 0 || readAt[0] != "novox/mesh-delivery@"+merge { + t.Errorf("read at %v, want the kept merge commit", readAt) + } + m, _ := moduleOf(t, mods, "g") + if m.Inputs2 == nil || !slices.Contains(m.Inputs2.Packages, conditions) { + t.Errorf("after: kept requirements = %+v, want %s", m.Inputs2, conditions) + } + // The recompute pins it at its published version, which moves the fingerprint (H-A-167, H-A-204), and the take-in again + // keeps the trunk's next revision of the same source, its call naming the repository, g its mover (H-A-212, H-A-213). + if m.Fp2 == "" || m.Fp2 == before.Fp2 || m.Packages[conditions] != "v0.1.4" { + t.Errorf("after: g = %s (%s) pinned %v, want a new fingerprint pinned to v0.1.4 (was %s)", m.Fp2, m.None2, m.Packages, before.Fp2) + } + rev, _, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil || !found || rev.Revision != 2 || rev.Source != 4 || rev.Call != "take-in again of novox/mesh-delivery" || + rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeInAgain || rev.Repository != "novox/mesh-delivery" || // H-A-215 + len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != m.Fp2 { + t.Errorf("the newest trunk revision = %+v (%v), want revision 2 of source 4 with the take-in again's call and g its mover", rev, err) + } + // H-A-217: a recompute that moves no fingerprint keeps nothing; one that moves one (a toolchain reported) keeps the trunk's + // next revision of the same source, its call-kind recompute, its call naming the reason, the moved modules its movers. + if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil { + t.Fatal(err) + } + if rev, _, _, _ := readNewestOf(ctx, ix.Revisions, TrunkBranch); rev.Revision != 2 { + t.Errorf("a recompute that moved nothing kept revision %d (%s), want none", rev.Revision, rev.Call) + } + reportToolchainRecipe(t, ix, "mesh-build-go", "novox", "go", "go1.27.1", "2") // a recipe change moves a Go module at once + moved, _ := moduleOf(t, mods, "g") + rev, _, found, err = readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil || !found || rev.Revision != 3 || rev.Source != 4 || rev.CallKind != meshcatalog.CatalogRevisionCallKindRecompute || + rev.Call != RecomputeCall("a toolchain reported") || !slices.ContainsFunc(rev.Movers, func(e meshcatalog.TakeInEntry) bool { + return e.Module == "g" && e.Fingerprint == moved.Fp2 && moved.Fp2 != m.Fp2 + }) { + t.Errorf("after a toolchain reported: the newest trunk revision = %+v (%v), want revision 3 of source 4, kind recompute, g its mover at %s (was %s)", + rev, err, moved.Fp2, m.Fp2) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 934e19c9a..3cba48d08 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -46,6 +46,8 @@ type TakeInAsk struct { HeldSource int64 `json:"held-source,omitempty"` Skip bool `json:"skip,omitempty"` NamedMember string `json:"named-member,omitempty"` + // Again names the repository taken in again at its kept merge commit (indexer_take_in_again, take_in_again.go; H-A-208, H-A-210). + Again string `json:"again,omitempty"` // RunDrop asks one run on request, by its group run-, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194). RunDrop string `json:"run-drop,omitempty"` // Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations, diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index 0b4d49474..667eb11a2 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -2,9 +2,11 @@ package index import ( "context" + "encoding/json" "errors" "fmt" "maps" + "slices" "strconv" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -25,9 +27,10 @@ func TakenID(branch string, source int64) string { // TakeInSource takes one source revision of a watched branch in, as the gate's source-moved names it (design 47 H §743): for each // repository it moves, the trees at the named commit through the seat git (wholeAt) and the fingerprints of the modules // registered for it; its movers (moversOf) against the branch's newest catalog revision; one catalog revision of the branch, -// ., carrying the movers and pinned for the take-in's pipeline when it has one (KeepBranchEntries); and +// the branch's next, . of source , carrying the movers and pinned for the take-in's pipeline when it has one +// (keepBranchEntries, H-A-213); and // take-in.taken once, with branch, source, revision, call and one entry per mover; a branch's first revision (start) has no -// mover (§768). A revision already kept under that name (a +// mover (§768). A revision already kept of that source (a // restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched // branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision. // An error leaves the source-moved held, written into the state held behind its branch's earlier ones (ErrHeldUnwritten when @@ -57,13 +60,13 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() - name := BranchObject(e.Branch, e.Source) - if kept, _, err := ReadRevision(ctx, ix.Revisions, name); err == nil { - taken.Revision, taken.Entries = kept.Revision, kept.Movers - ix.logf("take-in: %s is kept already: take-in.taken said again from its %d movers", name, len(kept.Movers)) - return taken, ix.sayTaken(ctx, taken) - } else if !errors.Is(err, ErrNoObject) { + // A source revision kept already is found by the revision's field source, never by its number (H-A-213 (1)). + if kept, found, err := takenOfSource(ctx, ix.Revisions, e.Branch, e.Source); err != nil { return taken, err + } else if found { + taken.Revision, taken.Entries = kept.Revision, kept.Movers + ix.logf("take-in: source revision %d of %s is kept already as %s: take-in.taken said again from its %d movers", e.Source, e.Branch, RevisionObject(kept), len(kept.Movers)) + return taken, ix.sayTaken(ctx, taken) } _, before, _, err := readNewestOf(ctx, ix.Revisions, e.Branch) if err != nil { @@ -106,14 +109,15 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE if e.Start { movers = nil // a branch's first revision moves nothing: its entries are the base later take-ins are held against (§768, H-A-19) } - if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil { + number, err := ix.keepBranchEntries(ctx, e.Branch, e.Source, takeInCall(e.Call), entries, movers) + if err != nil { if errors.Is(err, ErrObjectsFull) { ix.heldFull.Store(true) // room is said once the store has room again (RoomAgain), and wakes the take-in (§744) } return taken, err } - taken.Revision, taken.Entries = e.Source, movers - ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, name, len(e.Heads), len(movers)) + taken.Revision, taken.Entries = number, movers + ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, BranchObject(e.Branch, number), len(e.Heads), len(movers)) return taken, ix.sayTaken(ctx, taken) } @@ -157,22 +161,57 @@ func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelive if ix.Revisions == nil { return e, false, errors.New("the indexer keeps no catalog revisions") } - names, err := ix.Revisions.Names(ctx) + // The gate's source a branch is taken in at is its newest revision's field source, never the revision's number (H-A-213 (1)). + newest, found, err := newestOf(ctx, ix.Revisions, branch) if err != nil { - return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + return e, false, fmt.Errorf("the newest catalog revision of %s: %w", branch, err) } - var kept int64 - for _, n := range names { - if r, ok := BranchRevisionOf(branch, n); ok && r > kept { - kept = r - } - } - if kept >= a.Revision { + if found && newest.Source >= a.Revision { return e, false, nil } - e.Start = kept == 0 + e.Start = !found for _, r := range a.Repositories { e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head}) } return e, true, nil } + +// takenOfSource is the revision a take-in of source src kept on branch: of the branch's revisions whose field source is src, +// the lowest numbered (a re-pin or a take-in again keeps the same source on a later number, H-A-211, H-A-213). The revisions +// are read from the newest down while their source is not below src; none is false. +func takenOfSource(ctx context.Context, o Objects, branch string, src int64) (meshcatalog.CatalogRevision, bool, error) { + names, err := o.Names(ctx) + if err != nil { + return meshcatalog.CatalogRevision{}, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + } + var numbers []int64 + for _, n := range names { + if r, ok := BranchRevisionOf(branch, n); ok { + numbers = append(numbers, r) + } + } + slices.Sort(numbers) + var kept meshcatalog.CatalogRevision + found := false + for i := len(numbers) - 1; i >= 0; i-- { + name := BranchObject(branch, numbers[i]) + data, err := o.Get(ctx, name) + if errors.Is(err, ErrNoObject) { + continue // removed under the read by the pin judge + } + if err != nil { + return kept, false, fmt.Errorf("revision %s: %w", name, err) + } + var rev meshcatalog.CatalogRevision + if err := json.Unmarshal(data, &rev); err != nil { + return kept, false, fmt.Errorf("%s: %w", name, err) + } + if rev.Source < src { + break + } + if rev.Source == src { + kept, found = rev, true + } + } + return kept, found, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index d36933e00..2e439e85d 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -43,7 +43,8 @@ func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTa return ix, objects, said, f } -// Design 47 H §743, H-A-32, H-A-92: a take-in writes one catalog revision . carrying its movers, pins it for the +// Design 47 H §743, H-A-32, H-A-92, H-A-213 (1): a take-in writes one catalog revision ., n the branch's newest + 1 +// and its source the gate's, carrying its movers, pins it for the // take-in's pipeline and says take-in.taken once; a redelivery writes nothing again and says the same; a revision with no mover // pins nothing and says no entries; the old form is refused. func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { @@ -56,19 +57,20 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { t.Fatal(err) } if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Fingerprint == "" || taken.Entries[0].Commit != f.Merge || - taken.Revision != 7 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call { - t.Fatalf("taken %+v, want revision 7 of main with avahi as its one mover", taken) + taken.Revision != 1 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call { + t.Fatalf("taken %+v, want revision 1 (the branch's first) of source 7 of main with avahi as its one mover", taken) } - rev, _, err := ReadRevision(ctx, objects, "main.7") - if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 { - t.Fatalf("main.7 = %+v (%v), want it kept carrying the movers", rev, err) + rev, _, err := ReadRevision(ctx, objects, "main.1") + if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 || + rev.Call != moved.Call || rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeIn { // H-A-215 + t.Fatalf("main.1 = %+v (%v), want it kept carrying the movers", rev, err) } - if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 7)); !found { - t.Error("main.7 is not pinned for its pipeline") + if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 1)); !found { + t.Error("main.1 is not pinned for its pipeline") } else { var pv PinValue - if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.7"}) { - t.Errorf("main.7 is pinned by %s, want pipeline.main.7", v) + if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.1"}) { + t.Errorf("main.1 is pinned by %s, want pipeline.main.1", v) } } puts := objects.puts @@ -79,14 +81,14 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { same := moved same.Source, same.Call = 8, "fedcba9876543210" none, err := ix.TakeInSource(ctx, same) - if err != nil || len(none.Entries) != 0 || none.Revision != 8 { - t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 8 with no entries", none, err) + if err != nil || len(none.Entries) != 0 || none.Revision != 2 || none.Source != 8 { + t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 2 of source 8 with no entries", none, err) } - if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 8)); found { + if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 2)); found { t.Error("a take-in with no mover pinned its revision") } - if _, _, err := ReadRevision(ctx, objects, "main.8"); err != nil { - t.Errorf("main.8 is not kept: %v", err) + if _, _, err := ReadRevision(ctx, objects, "main.2"); err != nil { + t.Errorf("main.2 is not kept: %v", err) } if len(*said) != 3 { t.Errorf("said %d take-in.taken, want 3", len(*said)) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index f8c1eb677..dfba512a6 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -79,9 +79,22 @@ func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value return ix.ToolchainReported(ctx) } -// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute). +// ToolchainReported is what a changed toolchain moves: the modules written again, kept as a revision (RecomputeKept). func (ix *Indexer) ToolchainReported(ctx context.Context) error { - return ix.Recompute(ctx) + return ix.RecomputeKept(ctx, "a toolchain reported") +} + +// RecomputeKept is Recompute kept as a re-pin is (design 47 H-A-217): when it moved at least one fingerprint against the +// trunk's newest revision, it keeps the trunk's next revision of the same source, its movers the modules that moved, its +// call-kind recompute and its call naming reason, pinned for its pipeline, and says catalog-revision; one that moves none +// keeps nothing. Recompute writes the trunk's modules state, so only the trunk's revision is kept. +func (ix *Indexer) RecomputeKept(ctx context.Context, reason string) error { + ix.pendingMu.Lock() // one take-in step at a time, so the trunk's newest stays the one compared against (pending.go) + defer ix.pendingMu.Unlock() + if err := ix.Recompute(ctx); err != nil { + return err + } + return ix.keepRepinned(ctx, ix, TrunkBranch, recomputeCall(reason), nil) } // agreed is the toolchain of each language every reporting node agrees on, or why there is none. @@ -298,7 +311,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.fp, r.none = f.fp, f.none if r.fp != "" { // a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167) - r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository) + r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository, kept.Packages) r.tcs = toolchainsUsed(toolchains) } } @@ -517,7 +530,7 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, if f.none != "" || f.fp == "" { return "", nil, false } - fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository) + fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository, kept.Packages) return fp, pkgs, fp == k.held.Fp2 } @@ -544,7 +557,7 @@ func (ix *Indexer) RecomputeAtStart(ctx context.Context) error { ix.logf("no toolchain is read yet: the new-form fingerprints are computed when one is") return nil } - return ix.Recompute(ctx) + return ix.RecomputeKept(ctx, "the indexer's start") } // fillPaths gives every module whose key names no directory (a key written before the indexer learned paths) the one its @@ -598,29 +611,128 @@ func importsOf(pkgs sdkstep.PackageVersions, repository string) bool { return false } +// ReleaseCall is the call of the catalog revision a re-pin keeps: the release of a module at a version (design 47 H-A-213 (2)). +func ReleaseCall(module, version string) string { return "release of " + module + " " + version } + // Repin moves the pins of every module whose packages hold a package of module (a held module's directory holds it) to -// module's newest released version, the pass of H-A-161 for those modules alone: a version-released of module starts it -// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It answers how many it named. -func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) { +// module's newest released version, the pass of H-A-161 for those modules alone: a release of module at version starts it +// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It runs on every watched branch +// holding such a module: the first on the indexer's own modules state, another on its newest revision's picture (as a +// take-in does), and keeps on each whose re-pinned fingerprints moved the branch's next catalog revision, of the same +// source as its newest, with the call ReleaseCall and those modules as its movers, pinned for the release's pipeline, and +// says catalog-revision; no take-in.taken (H-A-211, H-A-212, H-A-213). It answers how many modules it named. +func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, error) { if ix.Modules == nil || ix.Manifests == nil { return 0, nil } - held, err := ix.heldModules(ctx) + ix.pendingMu.Lock() // one take-in step at a time, so the branch's newest stays the one re-pinned (pending.go) + defer ix.pendingMu.Unlock() + branches, err := ix.watchedBranches(ctx) if err != nil { return 0, err } + if !slices.Contains(branches, TrunkBranch) { + branches = append([]string{TrunkBranch}, branches...) // the modules state is re-pinned whether a revision is kept or not + } + named := 0 + for _, branch := range branches { + x := ix + if !isTrunkBranch(branch) { + view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) + if err != nil { + return named, err + } + d := ix.dry() + d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) + d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) } + x = d.ix + } + only, err := x.repinned(ctx, module) + if err != nil { + return named, err + } + if len(only) == 0 { + continue + } + named += len(only) + x.recompute.Lock() + err = x.recomputeLocked(ctx, nil, nil, "", only) + x.recompute.Unlock() + if err != nil { + return named, err + } + if err := ix.keepRepinned(ctx, x, branch, releaseCall(module, version), only); err != nil { + return named, err + } + } + return named, nil +} + +// repinned are the modules the indexer's modules state holds whose packages name a package of module: by their pins, and +// by their kept requirements, which name a package they hold no pin of yet (design 47 H-A-204, H-A-165, H-A-167). +func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool, error) { + held, err := ix.heldModules(ctx) + if err != nil { + return nil, err + } only := map[string]bool{} for key, m := range held { - for p := range m.Packages { + names := slices.Collect(maps.Keys(m.Packages)) + if m.Inputs2 != nil { + names = append(names, m.Inputs2.Packages...) + } + for _, p := range names { if moduleOfKey(fingerprint.HolderOf(p, held)) == module { only[key] = true } } } - if len(only) == 0 { - return 0, nil + return only, nil +} + +// keepRepinned keeps x's view, re-pinned for the modules only names (every module for nil), as branch's next catalog revision +// of the source of its newest, with call, when a fingerprint of them moved against that newest; its movers are those that +// moved (H-A-211, H-A-212, H-A-213). A branch with no revision kept yet keeps none: there is no source to keep it of. +func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch string, call revisionCall, only map[string]bool) error { + if ix.Revisions == nil { + return nil } - ix.recompute.Lock() - defer ix.recompute.Unlock() - return len(only), ix.recomputeLocked(ctx, nil, nil, "", only) + newest, before, found, err := readNewestOf(ctx, ix.Revisions, branch) + if err != nil { + return fmt.Errorf("the newest catalog revision of %s: %w", branch, err) + } + if !found { + ix.logf("re-pin (%s): %s keeps no revision yet, so none is kept for it", call.Call, branch) + return nil + } + entries, err := x.viewEntries(ctx) + if err != nil { + return err + } + was := make(map[string]string, len(before)) + for _, e := range before { + was[e.Module] = e.Fp + } + var movers []meshcatalog.TakeInEntry + moved := slices.Sorted(maps.Keys(only)) + if only == nil { // a take-in again or a recompute: every module whose fingerprint moved (H-A-212, H-A-217) + moved = slices.Sorted(maps.Keys(entries)) + } + for _, key := range moved { + e, ok := entries[key] + if !ok || e.Fp == "" || was[e.Module] == e.Fp { + continue + } + movers = append(movers, meshcatalog.TakeInEntry{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) + } + if len(movers) == 0 { + ix.logf("re-pin (%s): no fingerprint of %s moved, so no revision is kept", call.Call, branch) + return nil + } + number, err := ix.keepBranchEntries(ctx, branch, newest.Source, call, entries, movers) + if err != nil { + return err + } + ix.logf("re-pin (%s): %s kept as revision %d of source %d, %d movers", call.Call, branch, number, newest.Source, len(movers)) + return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 561b544a1..059aa977b 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -504,7 +504,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // } return first.KeepBranchEntries(context.Background(), TrunkBranch, source, entries, nil) } - if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 6 || + if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || firstSaid[0].Entries != int64(len(builderModules(t))) { t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err) } @@ -519,7 +519,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if err := takeIn(7); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) { t.Errorf("the next take-in after a write said %+v (%v), want revision 7 naming avahi", firstSaid, err) } - if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 7 { + if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 { t.Errorf("the view at a restart is read at %d (%v), want 7", at, err) } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { t.Errorf("the view at a restart lost the verdict: %s", raw) @@ -536,21 +536,21 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("the first-view tools %+v serve no modules", FirstViewTools) } if a, err := views.FirstView(testSelf, ModulesState, trunkStates.State(ModulesState), trunkStates.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 7 || len(a.Values) != len(builderModules(t)) { - t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 7", len(a.Values), a.Revision, err) + a.Revision != 2 || len(a.Values) != len(builderModules(t)) { + t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 2", len(a.Values), a.Revision, err) } else if m, err := meshcatalogclient.DecodeModule(a.Values[viewKey]); err != nil || m.Manifest == "" { t.Errorf("the first view's avahi = %s (%v), want the module's value naming its manifest", a.Values[viewKey], err) } - if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":7`) { - t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 7", e, ok, err) + if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) { + t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err) } // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never // one revision read per key (about 11 s a page live, 2026-11-14). reads := 0 keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 7 || len(a.Values) != len(builderModules(t)) || a.Next != "" { - t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 7 in one page", len(a.Values), a.Revision, a.Next, err) + a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { + t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) } if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 { t.Errorf("the kept view read the revision %d times (%v), want once", reads, err) @@ -822,7 +822,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.2.0", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil { t.Fatal(err) } - if _, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil { // the pass for avahi alone: mesh-tools is held stale here + if _, err := ix.Repin(context.Background(), "mesh-sdk", "0.1.27"); err != nil { // the pass for avahi alone: mesh-tools is held stale here t.Fatal(err) } if b, _ := moduleOf(t, mods, "avahi"); b.Fp2 != a.Fp2 || b.Packages[sdk] != "v0.1.18" { diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 9500d3025..e764bdad8 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -376,12 +376,20 @@ func isFloor(raw []byte) bool { // release number is never a pin. A package with none published in that series is kept with no version, so its kind-1 // edge stands (H §746) and its build waits for the publish (H-A-165). A module of the module's own repository is never // pinned, whatever a build reported: it is built from the tree (design 47 H-A-198, H-A-201, H-A-202). nil for none. -func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string) sdkstep.PackageVersions { - if len(versions) == 0 { +// +// required are the mesh packages the module's go.mod graph requires (its kept packages, H-A-201): each of another +// repository is named whether or not the form's versions name it, with no version while none is published, never left +// out, so the builder waits for its publish instead of the build module refusing (design 47 H-A-204, H-A-165). +func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) sdkstep.PackageVersions { + names := slices.Collect(maps.Keys(versions)) + if repository != "" { + names = append(names, fingerprint.OtherPackages(required, repository)...) + } + if len(names) == 0 { return nil } out := sdkstep.PackageVersions{} - for p := range versions { + for _, p := range names { if repo, _, ok := trunk.PackageRepository(p); ok && repository != "" && trunk.SameRepository(repo, repository) { continue } @@ -403,8 +411,8 @@ func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPack // form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover // built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that // stands is kept while its pins are the ones it was built with, else its import form takes the new pins. -func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string) (string, sdkstep.PackageVersions) { - pins := releasedPins(f.versions, series, m, repository) +func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) (string, sdkstep.PackageVersions) { + pins := releasedPins(f.versions, series, m, repository, required) if f.stands { if maps.Equal(f.built, map[string]string(pins)) { return f.fp, pins @@ -443,7 +451,7 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions { // repin is Repin of a release of module at version, said in the journal; its failure is the caller's error. func (ix *Indexer) repin(ctx context.Context, module, version string) error { - n, err := ix.Repin(ctx, module) + n, err := ix.Repin(ctx, module, version) if err != nil { return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err) } diff --git a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go index cdc1fc3fd..d7a136cd5 100644 --- a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go +++ b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go @@ -102,7 +102,7 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn( n := NewNamed(newMem(), nil) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} n.Changed = ix.RemoveUnlisted - if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { // the revision that listed it is removed @@ -213,7 +213,7 @@ func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testin fp := namedFp("c") setup := func(n *Named) (*Indexer, func() bool) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} - if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 8a24e6afc..81ef37ba3 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -205,9 +205,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.FingerprintsAgainTool, // issue 1966 // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, - index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || !slices.Contains(m.Tools, "modules") { + index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) + if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 0eeaa80c2..bc75c22c2 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -57,6 +57,7 @@ "indexer_recompute", "indexer_drop_candidate_graph", "indexer_entries", + "indexer_entry", "indexer_named_drop", "indexer_proposals", "indexer_proposal_drop", @@ -66,6 +67,7 @@ "indexer_pin_drop", "indexer_pending", "indexer_take_in_pending", + "indexer_take_in_again", "indexer_take_in_skip", "indexer_runs", "indexer_run_drop" diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index e6a0762bd..668d2ed80 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -206,7 +206,7 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) if raw == nil { return seat.Pipeline{}, false, nil } - p, err := seat.DecodePipelinesByBranchSourceValue(raw) + p, err := seat.DecodePipelinesByBranchRevisionValue(raw) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cfb8cc482..4c9c49cc8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -50,7 +50,7 @@ type Begin struct { // Key is a take-in's pipeline key in the state pipelines, `.`, the branch escaped as a key's name is // (design 47 H §746). func Key(branch string, source int64) string { - return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)) + return seat.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)) } // PublishedKey is the key of a pipeline a package's publish starts, `..published..`, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 019127c75..071981452 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -40,7 +40,7 @@ func TestStarts(t *testing.T) { if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Movers: []seat.Mover{{Module: "app"}}}); err != nil { t.Fatal(err) } - if b, r, ok := seat.ParsePipelinesByBranchSourceKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { + if b, r, ok := seat.ParsePipelinesByBranchRevisionKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { t.Errorf("the key of release.1 is %q", Key("release.1", 3)) } if _, err := s.Start(Begin{Branch: "main", Movers: []seat.Mover{{Module: "app"}}}); err == nil {