Merge commit '84989daf' into feat/47-H2-i

This commit is contained in:
2026-11-22 01:31:32 +01:00
40 changed files with 1847 additions and 249 deletions
@@ -14,13 +14,13 @@ const Seat = "mesh-catalog"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 2
const Form = 3
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
generatedForms = "2"
generatedFieldsDigest = "ActingRule=8a83ce7e0457 Artifact=78b05c1a9ec3 ArtifactPackage=2a3a660a934d ArtifactRef=a2f06147f4b6 BuildEdge=76f0684e254a BuildEdges=d2b95000259e BuildEdgesByModuleBranchValue=4fdf7cc1e723 CandidatesByGroupValue=9ea8c6c580d2 CandidatesByGroupValuePullsItem=90fb03643258 CandidatesByGroupValuePullsItemFpsValue=4c6783201aa3 CatalogEntry=d154f232237b CatalogRevision=d1ecaaa484af CatalogRevisionDefinitionsItem=77fcff620d48 CatalogRevisionEvent=710cbc5bfff1 CatalogRevisionsByBranchRevisionValue=17e54f635d35 CatalogRevisionsByCandidateGroupHeadsValue=17e54f635d35 CatalogRevisionsByEntryDigestValue=8755036a3051 DependentsAnswer=7538f2f5e68e Edge=7242732afaef EntryAnswer=805c2ff28e1c EntryInputs=76a7f8690dd2 EntryRef=ff109e5089e1 Fill=80f442916429 FillingEvent=5bd4df78ed84 FingerprintInputs=99a6df7f991f FullEvent=5bd4df78ed84 Head=774b4bf88ba4 InputAnswer=cd41ee65565d InputsByTimeDigestValue=a2f06147f4b6 ManifestsByDigestValue=51b3822b29c6 ModuleAnswer=499a03ce9b27 ModuleAnswerModule=c0f679e09f08 ModulesAnswer=6d119571c69e ModulesAnswerModulesItem=a70f39c60b67 ModulesByModuleValue=fc7530ba893a ModulesByModuleValueInputs2=99a6df7f991f Outdated=e820214f66bc OutdatedAnswer=275fb25266b9 OutdatedDependency=dc35dca1b683 PackagesByModuleValue=2b62574b709f PackagesByModuleValuePackagesItem=a7d30558470d PackagesByPackageSeriesValue=07421d8d3411 PackagesByPackageSeriesValueNewest=aba144994370 PackagesByPackageSeriesValueNewestContents=eca2ccf56913 PackagesByPackageSeriesValueNewestContentsPackagesItem=a7d30558470d PackagesByPackageSeriesValueNumbered=c8ffc628599c PackagesByPackageSeriesValueNumberedContents=0def6c0794ee PackagesByPackageSeriesValueNumberedContentsPackagesItem=a7d30558470d Policy=719e2f460c6f PolicyAnswer=46cdf8430557 PredefinedByListValue=ec8e3febe8ac ProvidesAnswer=7538f2f5e68e RegisterAnswer=46cdf8430557 Registration=ef03d1b2e73f RegistrationAsked=55adba349c24 RegistrationChangedEvent=99a3268edf41 RegistrationHistoryByModuleVersionValue=0999b5bb7dc7 RegistrationHistoryBySourceOwnerNameVersionValue=0999b5bb7dc7 RegistrationsAnswer=8849ef618cf3 RegistrationsByModuleValue=0999b5bb7dc7 RegistrationsBySourceOwnerNameValue=0999b5bb7dc7 RevisionAnswer=258ea5124931 RoomEvent=5bd4df78ed84 SeatDefinitionChangedEvent=e271d24d58f7 SeatDefinitionRemovedEvent=f7d6171111e6 SeatDefinitionsAnswer=0fd6a59fec59 SeatDefinitionsAnswerEntry=d006f68d2be5 SeatDefinitionsAnswerSeatsItem=e271d24d58f7 SeatListAnswer=912c0cebcf0a SourceCheck=d1ed7cbbd579 StepStatuses=4f50aa633e61 TakeInEntry=545ec1528499 ToolchainUsed=43b6d42376fb UnwatchAnswer=46cdf8430557 VersionArtifact=438c49a392b0 VersionEdge=b0f80e83f4dc VersionEntry=33f5b00dbda9 VersionReleasedEvent=4ccf1ea90e7f VersionsAnswer=b59e6e057e7b VersionsByModuleVersionValue=e546dfadb913 VersionsFilledEvent=4cc437912dc3 WatchAnswer=46cdf8430557"
generatedForms = "3"
generatedFieldsDigest = "ActingRule=8a83ce7e0457 Artifact=78b05c1a9ec3 ArtifactPackage=2a3a660a934d ArtifactRef=a2f06147f4b6 BuildEdge=76f0684e254a BuildEdges=d2b95000259e BuildEdgesByModuleBranchValue=4fdf7cc1e723 CandidatesByGroupValue=9ea8c6c580d2 CandidatesByGroupValuePullsItem=90fb03643258 CandidatesByGroupValuePullsItemFpsValue=4c6783201aa3 CatalogEntry=c0a4ea25d523 CatalogRevision=d1ecaaa484af CatalogRevisionDefinitionsItem=77fcff620d48 CatalogRevisionEvent=710cbc5bfff1 CatalogRevisionsByBranchRevisionValue=17e54f635d35 CatalogRevisionsByCandidateGroupHeadsValue=17e54f635d35 CatalogRevisionsByEntryDigestValue=8755036a3051 DependentsAnswer=7538f2f5e68e Edge=7242732afaef EntryAnswer=805c2ff28e1c EntryInputs=76a7f8690dd2 EntryRef=ff109e5089e1 Fill=80f442916429 FillingEvent=5bd4df78ed84 FingerprintInputs=99a6df7f991f FullEvent=5bd4df78ed84 Head=774b4bf88ba4 InputAnswer=cd41ee65565d InputsByTimeDigestValue=a2f06147f4b6 ManifestsByDigestValue=51b3822b29c6 ModuleAnswer=499a03ce9b27 ModuleAnswerModule=c0f679e09f08 ModulesAnswer=6d119571c69e ModulesAnswerModulesItem=a70f39c60b67 ModulesByModuleValue=fc7530ba893a ModulesByModuleValueInputs2=99a6df7f991f Outdated=e820214f66bc OutdatedAnswer=275fb25266b9 OutdatedDependency=dc35dca1b683 PackagesByModuleValue=2b62574b709f PackagesByModuleValuePackagesItem=a7d30558470d PackagesByPackageSeriesValue=07421d8d3411 PackagesByPackageSeriesValueNewest=aba144994370 PackagesByPackageSeriesValueNewestContents=eca2ccf56913 PackagesByPackageSeriesValueNewestContentsPackagesItem=a7d30558470d PackagesByPackageSeriesValueNumbered=c8ffc628599c PackagesByPackageSeriesValueNumberedContents=0def6c0794ee PackagesByPackageSeriesValueNumberedContentsPackagesItem=a7d30558470d Policy=719e2f460c6f PolicyAnswer=46cdf8430557 PredefinedByListValue=ec8e3febe8ac ProvidesAnswer=7538f2f5e68e RegisterAnswer=46cdf8430557 Registration=ef03d1b2e73f RegistrationAsked=55adba349c24 RegistrationChangedEvent=99a3268edf41 RegistrationHistoryByModuleVersionValue=0999b5bb7dc7 RegistrationHistoryBySourceOwnerNameVersionValue=0999b5bb7dc7 RegistrationsAnswer=8849ef618cf3 RegistrationsByModuleValue=0999b5bb7dc7 RegistrationsBySourceOwnerNameValue=0999b5bb7dc7 RevisionAnswer=258ea5124931 RoomEvent=5bd4df78ed84 SeatDefinitionChangedEvent=e271d24d58f7 SeatDefinitionRemovedEvent=f7d6171111e6 SeatDefinitionsAnswer=0fd6a59fec59 SeatDefinitionsAnswerEntry=d006f68d2be5 SeatDefinitionsAnswerSeatsItem=e271d24d58f7 SeatListAnswer=912c0cebcf0a SourceCheck=d1ed7cbbd579 StepStatuses=4f50aa633e61 TakeInEntry=545ec1528499 ToolchainUsed=43b6d42376fb UnwatchAnswer=46cdf8430557 VersionArtifact=438c49a392b0 VersionEdge=b0f80e83f4dc VersionEntry=33f5b00dbda9 VersionReleasedEvent=4ccf1ea90e7f VersionsAnswer=b59e6e057e7b VersionsByModuleVersionValue=e546dfadb913 VersionsFilledEvent=4cc437912dc3 WatchAnswer=46cdf8430557"
)
const (
@@ -679,6 +679,8 @@ type CatalogEntry struct {
Awaits []string `json:"awaits,omitempty"`
// the images and modules it is built on
Bases []string `json:"bases,omitempty"`
// core or leaf: core when its manifest claims a seat, leaf when it claims none; an entry without it reads as leaf (design 47 H2 H-A-394, H-A-431)
Class string `json:"class,omitempty"`
// the merge commit it was last changed at
Commit string `json:"commit"`
// the requirements the manifest's contributes object names as keys, sorted, its local names not carried; each is also in requires (design 47 G section 709)
@@ -83,6 +83,12 @@ func TestFixtures(t *testing.T) {
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
{name: "a seat-claiming module's entry carries its class core (H-A-394, H-A-431)", subject: "", given: "{\"acts\":[{\"bound\":\"60m\",\"on\":\"go\",\"seat\":\"node-build-go\",\"step\":\"build\"}],\"awaits\":[\"database\"],\"class\":\"core\",\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"left-out\":[\"mesh-catalog.revision\"],\"main\":\"cmd/mesh-build-go\",\"manifest\":{\"module\":\"mesh-build-go\",\"version\":\"1\"},\"module\":\"mesh-build-go\",\"needs\":[\"form:setting\",\"word:claims\"],\"path\":\"modules/mesh-build-go\",\"repository\":\"novox/mesh-catalog\",\"serves\":[\"node-build-go.run\"],\"v\":1,\"verdict\":\"accepted\"}", wire: "{\"acts\":[{\"bound\":\"60m\",\"on\":\"go\",\"seat\":\"node-build-go\",\"step\":\"build\"}],\"awaits\":[\"database\"],\"class\":\"core\",\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"left-out\":[\"mesh-catalog.revision\"],\"main\":\"cmd/mesh-build-go\",\"manifest\":{\"module\":\"mesh-build-go\",\"version\":\"1\"},\"module\":\"mesh-build-go\",\"needs\":[\"form:setting\",\"word:claims\"],\"path\":\"modules/mesh-build-go\",\"repository\":\"novox/mesh-catalog\",\"serves\":[\"node-build-go.run\"],\"v\":1,\"verdict\":\"accepted\"}",
decode: func(b []byte) (any, error) {
var v CatalogRevisionsByEntryDigestValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
{name: "a module's registration", subject: "", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"module\":\"nats\",\"origin\":\"novox/mesh-catalog\",\"path\":\"modules/nats/module.json\",\"policy\":{\"artifacts\":{\"program\":{\"unit\":\"bypass\"}},\"steps\":{\"approval\":\"bypass\",\"composition\":\"required\",\"integration\":\"required\",\"lint\":\"required\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":2,\"warrant\":\"ask-m2kx9a1b3c4d5e\"}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"module\":\"nats\",\"origin\":\"novox/mesh-catalog\",\"path\":\"modules/nats/module.json\",\"policy\":{\"artifacts\":{\"program\":{\"unit\":\"bypass\"}},\"steps\":{\"approval\":\"bypass\",\"composition\":\"required\",\"integration\":\"required\",\"lint\":\"required\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":2,\"warrant\":\"ask-m2kx9a1b3c4d5e\"}",
decode: func(b []byte) (any, error) {
var v RegistrationsByModuleValue
@@ -14,32 +14,33 @@ const Seat = "mesh-pipeline-runner"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 2
const Form = 3
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
generatedForms = "2"
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b5c78894cc5b Member=8a103baec01f Mover=5659e54b6dd0 MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=5beff763f531 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepOutputsByPipelineStepArtifactValue=a2f06147f4b6 StepResult=94c2bea6ddcd StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
generatedForms = "3"
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b5c78894cc5b Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=5beff763f531 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepOutputsByPipelineStepArtifactValue=a2f06147f4b6 StepResult=94c2bea6ddcd StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
)
const (
VerbRun = "run"
VerbStart = "start"
VerbShow = "show"
VerbList = "list"
VerbStands = "stands"
VerbWhyCarried = "why-carried"
VerbKept = "kept"
VerbReview = "review"
VerbRerun = "rerun"
VerbAccept = "accept"
VerbCancel = "cancel"
VerbEndAgain = "end-again"
VerbPlace = "place"
VerbAskAgain = "ask-again"
VerbTestRuns = "test-runs"
VerbTestRunsDrop = "test-runs-drop"
VerbRun = "run"
VerbStart = "start"
VerbShow = "show"
VerbList = "list"
VerbStands = "stands"
VerbWhyCarried = "why-carried"
VerbKept = "kept"
VerbReview = "review"
VerbRerun = "rerun"
VerbAccept = "accept"
VerbCancel = "cancel"
VerbEndAgain = "end-again"
VerbPlace = "place"
VerbPlaceUnfinished = "place-unfinished"
VerbAskAgain = "ask-again"
VerbTestRuns = "test-runs"
VerbTestRunsDrop = "test-runs-drop"
)
const (
@@ -58,7 +59,7 @@ const (
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
func Verbs() []string {
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbEndAgain, VerbPlace, VerbAskAgain, VerbTestRuns, VerbTestRunsDrop}
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbEndAgain, VerbPlace, VerbPlaceUnfinished, VerbAskAgain, VerbTestRuns, VerbTestRunsDrop}
}
// Events is every event published on the seat's subject: a new copy each call.
@@ -274,6 +275,44 @@ func DecodePipelinesByBranchRunRunValue(raw []byte) (PipelinesByBranchRunRunValu
return v, formRead(raw), nil
}
// PipelinesByBranchRevisionUnfinishedModuleKey is the key of one value of the state pipelines under the form "<branch>.<revision>.unfinished.<module>".
func PipelinesByBranchRevisionUnfinishedModuleKey(branch, revision, module string) string {
return branch + "." + revision + ".unfinished." + module
}
// ParsePipelinesByBranchRevisionUnfinishedModuleKey reads the parts of a key of the form "<branch>.<revision>.unfinished.<module>", in the order of the form; the last result is false for a key of another form.
func ParsePipelinesByBranchRevisionUnfinishedModuleKey(key string) (string, string, string, bool) {
rest := key
v0, after0, found0 := strings.Cut(rest, ".")
if !found0 || v0 == "" {
return "", "", "", false
}
rest = after0
v2, after2, found2 := strings.Cut(rest, ".unfinished.")
if !found2 || v2 == "" {
return "", "", "", false
}
rest = after2
if rest == "" {
return "", "", "", false
}
v4 := rest
rest = ""
if rest != "" {
return "", "", "", false
}
return v0, v2, v4, true
}
// DecodePipelinesByBranchRevisionUnfinishedModuleValue reads one value of the state pipelines under the form "<branch>.<revision>.unfinished.<module>" as another part wrote it: a field PipelinesByBranchRevisionUnfinishedModuleValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePipelinesByBranchRevisionUnfinishedModuleValue(raw []byte) (PipelinesByBranchRevisionUnfinishedModuleValue, int, error) {
var v PipelinesByBranchRevisionUnfinishedModuleValue
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// PipelinesByStepsPipelineMoverKey is the key of one value of the state pipelines under the form "steps.<pipeline>.<mover>".
func PipelinesByStepsPipelineMoverKey(pipeline, mover string) string {
return "steps." + pipeline + "." + mover
@@ -598,16 +637,25 @@ type Mover struct {
// yes when the module is over its test budget (design 47 G section 419)
OverBudget bool `json:"over-budget,omitempty"`
// the artifact references, <module>:<artifact>, of the mover's dependencies whose package-published (and, for a kind-3 edge, whose build step's passed end) was heard at the pipeline's branch and source, written with the pipeline by compare-and-set; absent when none (H-A-84)
Published []string `json:"published,omitempty"`
Repository string `json:"repository"`
Tests *TestCounts `json:"tests,omitempty"`
Published []string `json:"published,omitempty"`
Repository string `json:"repository"`
// the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787)
Revision int64 `json:"revision,omitempty"`
Tests *TestCounts `json:"tests,omitempty"`
// the mover's tier in the pipeline's delivery order, from 1 (ADR 0666 section 14; design 47 H §746)
Tier int64 `json:"tier"`
// why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed
Unplaced string `json:"unplaced,omitempty"`
// the version released for the mover in this pipeline (version-released), absent until one is (design 47 H-A-52)
Version string `json:"version,omitempty"`
Widens []Widening `json:"widens,omitempty"`
}
// The values of Mover's unplaced.
const (
MoverUnplacedKeptRevisionUnread = "kept-revision-unread"
)
// MoverEnded is one mover at its pipeline's end (design 47 H §746).
type MoverEnded struct {
// the commit the mover's entry was last changed at, from the pipeline's value (design 47 H-A-119)
@@ -1383,6 +1431,22 @@ type PlaceAnswer struct {
Pipeline Pipeline `json:"pipeline"`
}
// PlaceUnfinishedInput is what place-unfinished takes.
type PlaceUnfinishedInput struct {
// the unfinished entry's key, <branch>.<module>
Key string `json:"key"`
// why, written with the caller the runtime proves in the hand-act log
Why string `json:"why"`
}
// PlaceUnfinishedAnswer is what place-unfinished answers.
type PlaceUnfinishedAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the key of the pipeline the mover is put in
Pipeline string `json:"pipeline"`
}
// AskAgainInput is what ask-again takes.
type AskAgainInput struct {
// the mover's module, so a pipeline of several movers is never ambiguous; a call naming none on a pipeline of one mover is taken as that mover (design 47 H-A-101)
@@ -1493,6 +1557,9 @@ type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline
// PipelinesByBranchRunRunValue is one value of the state pipelines under the key <branch>.run.<run>.
type PipelinesByBranchRunRunValue = Pipeline
// PipelinesByBranchRevisionUnfinishedModuleValue is one value of the state pipelines under the key <branch>.<revision>.unfinished.<module>.
type PipelinesByBranchRevisionUnfinishedModuleValue = Pipeline
// PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps.<pipeline>.<mover>.
type PipelinesByStepsPipelineMoverValue struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
@@ -1643,6 +1710,15 @@ func DecodePlaceAnswer(raw []byte) (PlaceAnswer, int, error) {
return v, formRead(raw), nil
}
// DecodePlaceUnfinishedAnswer reads a PlaceUnfinishedAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePlaceUnfinishedAnswer(raw []byte) (PlaceUnfinishedAnswer, int, error) {
var v PlaceUnfinishedAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeAskAgainAnswer reads a AskAgainAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeAskAgainAnswer(raw []byte) (AskAgainAnswer, int, error) {
var v AskAgainAnswer
@@ -345,6 +345,46 @@
]
}
},
{
"name": "a seat-claiming module's entry carries its class core (H-A-394, H-A-431)",
"of": "state:catalog-revisions/entry.<digest>",
"given": {
"v": 1,
"module": "mesh-build-go",
"repository": "novox/mesh-catalog",
"path": "modules/mesh-build-go",
"commit": "f469eb387a3015569d9b16c66ed418e9dd098f64",
"manifest": {
"module": "mesh-build-go",
"version": "1"
},
"verdict": "accepted",
"fp": "src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56",
"serves": [
"node-build-go.run"
],
"left-out": [
"mesh-catalog.revision"
],
"needs": [
"form:setting",
"word:claims"
],
"awaits": [
"database"
],
"main": "cmd/mesh-build-go",
"acts": [
{
"seat": "node-build-go",
"step": "build",
"on": "go",
"bound": "60m"
}
],
"class": "core"
}
},
{
"name": "a module's registration",
"of": "state:registrations/<module>",
@@ -1686,6 +1686,10 @@
"type": "string",
"description": "its manifest's verdict: accepted, refused or held"
},
"class": {
"type": "string",
"description": "core or leaf: core when its manifest claims a seat, leaf when it claims none; an entry without it reads as leaf (design 47 H2 H-A-394, H-A-431)"
},
"fp": {
"type": "string",
"description": "its fingerprint (src2: and 64 hex digits); absent when it has none"
@@ -2721,5 +2725,5 @@
]
}
},
"form": 2
"form": 3
}
@@ -578,6 +578,40 @@
},
"class": "act"
},
{
"name": "place-unfinished",
"description": "A mover kept in the state unfinished put in a pipeline of its own at once, keyed <branch>.<revision>.unfinished.<module>; written in the hand-act log (design 47 H2 §786, H-A-398).",
"input": {
"type": "object",
"properties": {
"key": {
"type": "string",
"description": "the unfinished entry's key, <branch>.<module>"
},
"why": {
"type": "string",
"description": "why, written with the caller the runtime proves in the hand-act log"
}
},
"required": [
"key",
"why"
]
},
"answer": {
"type": "object",
"properties": {
"pipeline": {
"type": "string",
"description": "the key of the pipeline the mover is put in"
}
},
"required": [
"pipeline"
]
},
"class": "act"
},
{
"name": "ask-again",
"description": "A declined approval step asked again (ADR 0666 section 10). Written in the hand-act log.",
@@ -858,6 +892,13 @@
},
"description": "a run on request's pipeline, one per run, its movers the named commits' modules; it delivers nothing (H-A-83)"
},
{
"key": "<branch>.<revision>.unfinished.<module>",
"value": {
"$ref": "#/defs/Pipeline"
},
"description": "a pipeline of its own of one mover kept in the runner's state unfinished, put in it by place-unfinished; the revision the unfinished entry names (design 47 H2 §86, H-A-398)"
},
{
"key": "steps.<pipeline>.<mover>",
"value": {
@@ -2108,6 +2149,17 @@
"tests": {
"$ref": "#/defs/TestCounts"
},
"unplaced": {
"type": "string",
"enum": [
"kept-revision-unread"
],
"description": "why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed"
},
"revision": {
"type": "integer",
"description": "the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787)"
},
"over-budget": {
"type": "boolean",
"description": "yes when the module is over its test budget (design 47 G section 419)"
@@ -2181,5 +2233,5 @@
]
}
},
"form": 2
"form": 3
}
@@ -236,6 +236,8 @@ export interface CatalogEntry {
"awaits"?: string[];
/** the images and modules it is built on */
"bases"?: string[];
/** core or leaf: core when its manifest claims a seat, leaf when it claims none; an entry without it reads as leaf (design 47 H2 H-A-394, H-A-431) */
"class"?: string;
/** the merge commit it was last changed at */
"commit": string;
/** the requirements the manifest's contributes object names as keys, sorted, its local names not carried; each is also in requires (design 47 G section 709) */
@@ -18,6 +18,7 @@ export declare const verbs: {
readonly cancel: "cancel";
readonly endAgain: "end-again";
readonly place: "place";
readonly placeUnfinished: "place-unfinished";
readonly askAgain: "ask-again";
readonly testRuns: "test-runs";
readonly testRunsDrop: "test-runs-drop";
@@ -115,6 +116,12 @@ export declare function pipelinesByBranchRunRunKey(branch: string, run: string):
/** The parts of a key of the form "<branch>.run.<run>", or undefined for a key of another form. */
export declare function parsePipelinesByBranchRunRunKey(_key: string): { branch: string; run: string } | undefined;
/** The key of one value of the state pipelines under the form "<branch>.<revision>.unfinished.<module>". */
export declare function pipelinesByBranchRevisionUnfinishedModuleKey(branch: string, revision: string, module: string): string;
/** The parts of a key of the form "<branch>.<revision>.unfinished.<module>", or undefined for a key of another form. */
export declare function parsePipelinesByBranchRevisionUnfinishedModuleKey(_key: string): { branch: string; revision: string; module: string } | undefined;
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
export declare function pipelinesByStepsPipelineMoverKey(pipeline: string, mover: string): string;
@@ -231,9 +238,13 @@ export interface Mover {
/** the artifact references, <module>:<artifact>, of the mover's dependencies whose package-published (and, for a kind-3 edge, whose build step's passed end) was heard at the pipeline's branch and source, written with the pipeline by compare-and-set; absent when none (H-A-84) */
"published"?: string[];
"repository": string;
/** the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787) */
"revision"?: number;
"tests"?: TestCounts;
/** the mover's tier in the pipeline's delivery order, from 1 (ADR 0666 section 14; design 47 H §746) */
"tier": number;
/** why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed */
"unplaced"?: "kept-revision-unread";
/** the version released for the mover in this pipeline (version-released), absent until one is (design 47 H-A-52) */
"version"?: string;
"widens"?: Widening[];
@@ -863,6 +874,22 @@ export interface PlaceAnswer {
"pipeline": Pipeline;
}
/** is what place-unfinished takes. */
export interface PlaceUnfinishedInput {
/** the unfinished entry's key, <branch>.<module> */
"key": string;
/** why, written with the caller the runtime proves in the hand-act log */
"why": string;
}
/** is what place-unfinished answers. */
export interface PlaceUnfinishedAnswer {
/** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */
"form"?: number;
/** the key of the pipeline the mover is put in */
"pipeline": string;
}
/** is what ask-again takes. */
export interface AskAgainInput {
/** the mover's module, so a pipeline of several movers is never ambiguous; a call naming none on a pipeline of one mover is taken as that mover (design 47 H-A-101) */
@@ -966,6 +993,9 @@ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline;
/** is one value of the state pipelines under the key <branch>.run.<run>. */
export type PipelinesByBranchRunRunValue = Pipeline;
/** is one value of the state pipelines under the key <branch>.<revision>.unfinished.<module>. */
export type PipelinesByBranchRevisionUnfinishedModuleValue = Pipeline;
/** is one value of the state pipelines under the key steps.<pipeline>.<mover>. */
export interface PipelinesByStepsPipelineMoverValue {
/** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */
@@ -18,6 +18,7 @@ export const verbs = {
cancel: "cancel",
endAgain: "end-again",
place: "place",
placeUnfinished: "place-unfinished",
askAgain: "ask-again",
testRuns: "test-runs",
testRunsDrop: "test-runs-drop",
@@ -176,6 +177,29 @@ export function parsePipelinesByBranchRunRunKey(_key) {
return { branch, run };
}
/** The key of one value of the state pipelines under the form "<branch>.<revision>.unfinished.<module>". */
export function pipelinesByBranchRevisionUnfinishedModuleKey(branch, revision, module) {
return branch + "." + revision + ".unfinished." + module;
}
/** The parts of a key of the form "<branch>.<revision>.unfinished.<module>", or undefined for a key of another form. */
export function parsePipelinesByBranchRevisionUnfinishedModuleKey(_key) {
let _rest = _key;
const _at0 = _rest.indexOf(".");
if (_at0 <= 0) return undefined;
const branch = _rest.slice(0, _at0);
_rest = _rest.slice(_at0 + 1);
const _at2 = _rest.indexOf(".unfinished.");
if (_at2 <= 0) return undefined;
const revision = _rest.slice(0, _at2);
_rest = _rest.slice(_at2 + 12);
if (_rest === "") return undefined;
const module = _rest;
_rest = "";
if (_rest !== "") return undefined;
return { branch, revision, module };
}
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
export function pipelinesByStepsPipelineMoverKey(pipeline, mover) {
return "steps." + pipeline + "." + mover;
@@ -13,6 +13,7 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/busstate"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/views"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
@@ -102,28 +103,41 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now
// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A
// running pipeline it first names stalled is said in this program's journal through logf, once per key for the
// program's life (H-A-311 (3), H-A-313).
func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool {
func HealthTool(pipelines, consumers, firstViews pipeline.Kept, bound time.Duration, now func() time.Time, logf func(string, ...any)) stdio.Tool {
var mu sync.Mutex
said := map[string]bool{}
return stdio.Tool{Name: Prefix + "_health",
Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " +
"unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " +
"newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key.",
"newly placed, nor a step's ask stayed untaken, for the setting stalled-bound (five minutes by default); otherwise it " +
"names each consumer behind, each such pipeline by its key and each such step. It " +
"answers registrations-read: whether the runner's registrations view's first read is in, and since when.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
a, stalled := pipeline.HealthStalled(pipelines, consumers, now())
a, stalled := pipeline.HealthStalled(pipelines, consumers, now(), bound)
// design 47 H2 §788: whether the registrations are read, from the runner's own first-views state.
read, err := pipeline.RegistrationsReadOf(firstViews)
if err != nil {
return nil, err
}
mu.Lock()
defer mu.Unlock()
for _, key := range stalled {
if !said[key] && logf != nil {
said[key] = true
logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter)
logf("%s has stood still for %s or more (nothing open and nothing newly placed, or its ask untaken): runner_health names it unhealthy (H-A-311, design 47 H2 §790)", key, bound)
}
}
return a, nil
return HealthAnswer{Answer: a, RegistrationsRead: read}, nil
}}
}
// HealthAnswer is runner_health's answer: the health and registrations-read (design 47 H2 §788).
type HealthAnswer struct {
health.Answer
RegistrationsRead pipeline.RegistrationsRead `json:"registrations-read"`
}
// StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states
// pipelines and kept by `mesh/state.info`, step-outputs by the object store's fill and step-log by the log's; the runner's
// process writes nothing for it. A store whose fill cannot be read is an error naming it.
@@ -161,6 +175,12 @@ func StoresFill(state func(string) (*stdio.StateInfo, error), objects func(strin
}
func main() {
// design 47 H2 §790, §781: the stalled bound is the runner's setting, read here as its process reads it.
stalledBound, err := stdio.SettingDuration(pipeline.StalledBoundSetting, pipeline.DefaultStalledBound)
if err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
os.Exit(1)
}
fill := StoresFill(func(n string) (*stdio.StateInfo, error) { return stdio.State(n).Info() },
func(n string) (buscalls.Fill, error) { return busstate.ObjectsOf(n).Fill() },
func(n string) (*stdio.LogInfo, error) { return stdio.Log(n).Info() }, time.Now)
@@ -168,12 +188,14 @@ func main() {
tools := append(Tools(r), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask},
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...)
tools = append(tools, ShowTools(r)...)
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
tools = append(tools, VerbTools(stdio.Ask)...)
tools = append(tools, TestRunsTools(stdio.Ask)...)
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now,
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, kept{stdio.State(views.StatusState)}, stalledBound, time.Now,
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) }))
tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)}))
tools = append(tools, UnfinishedTools(keptDelete{kept{stdio.State(pipeline.StateUnfinished)}}, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
os.Exit(1)
@@ -33,17 +33,27 @@ func TestTools(t *testing.T) {
for _, tl := range FirstViewTools("the-runner", stdio.State(views.StatusState), nil, nil) {
names = append(names, tl.Name)
}
// design 47 H2 §789: a pipeline read in parts.
for _, tl := range ShowTools(pipeline.Reads{Pipelines: none{}, Kept: none{}}) {
names = append(names, tl.Name)
}
names = append(names, CancelTool(nil, nil).Name, RerunTool(nil).Name)
for _, tl := range VerbTools(nil) {
names = append(names, tl.Name)
}
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
names = append(names, HealthTool(none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name)
names = append(names, HealthTool(none{}, none{}, none{}, pipeline.DefaultStalledBound, nil, nil).Name, WaitingTool(none{}).Name)
// §606, §549: the unlogged entries are listed and appended by hand.
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
names = append(names, tl.Name)
}
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_cancel runner_rerun runner_start runner_run runner_accept runner_place runner_end_again runner_ask_again runner_health runner_waiting runner_unlogged runner_unlogged_append"
// design 47 H2 §786: the unfinished movers are read and dropped by hand.
unfinished := unfinishedOfMem{"main.app": `{"fingerprint":"src2:aa","revision":4,"pipeline":"main.4","outcome":"failed"}`}
tools := UnfinishedTools(unfinished, nil)
for _, tl := range tools {
names = append(names, tl.Name)
}
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_show runner_movers runner_cancel runner_rerun runner_start runner_run runner_accept runner_place runner_end_again runner_unfinished_place runner_ask_again runner_health runner_waiting runner_unlogged runner_unlogged_append runner_unfinished runner_unfinished_drop"
if got := strings.Join(names, " "); got != want {
t.Errorf("tools %q, want %q", got, want)
}
@@ -60,7 +70,8 @@ func TestTools(t *testing.T) {
quiet := queueOf{"main.137": `{"pipeline":"main.137","branch":"main","outcome":"running","source":137,"started":"2026-11-21T20:15:00Z",` +
`"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`}
var journal []string
health := HealthTool(quiet, none{}, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
firstViews := queueOf{}
health := HealthTool(quiet, none{}, firstViews, pipeline.DefaultStalledBound, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
for range 2 {
out, err := health.Run(nil)
if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") {
@@ -70,8 +81,62 @@ func TestTools(t *testing.T) {
if len(journal) != 1 || !strings.Contains(journal[0], "main.137") {
t.Errorf("the journal said %q, want main.137 said once", journal)
}
// design 47 H2 §788: registrations-read is false, since the ask, until the registrations' first view is answered.
for _, c := range []struct{ status, want string }{
{`{"owner":"mesh-catalog","state":"registrations","asked":"2026-11-21T20:30:00Z"}`, `"registrations-read":{"read":false,"since":"2026-11-21T20:30:00Z"}`},
{`{"owner":"mesh-catalog","state":"registrations","asked":"2026-11-21T20:30:00Z","answered":"2026-11-21T20:30:02Z"}`, `"registrations-read":{"read":true,"since":"2026-11-21T20:30:02Z"}`},
} {
firstViews["mesh-catalog.registrations"] = c.status
out, err := health.Run(nil)
if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), c.want) {
t.Errorf("runner_health = %s (%v), want %s", got, err, c.want)
}
}
// design 47 H2 §786: runner_unfinished reads each key and its entry; runner_unfinished_drop names why, drops nothing when
// its hand act is not recorded, and drops the key once it is.
if got, err := tools[0].Run(nil); err != nil || fmt.Sprint(got) != "[{main.app {src2:aa 4 main.4 failed}}]" {
t.Errorf("runner_unfinished = %v (%v)", got, err)
}
for _, c := range []struct {
name string
acts views.HandActs
args map[string]any
refusal string
}{
{"no why", recordingActs{}, map[string]any{"key": "main.app"}, "why"},
{"no such key", recordingActs{}, map[string]any{"key": "main.lib", "why": "x"}, "holds no main.lib"},
{"not recorded", refusingActs{}, map[string]any{"key": "main.app", "why": "x"}, "not recorded"},
{"dropped", recordingActs{}, map[string]any{"key": "main.app", "why": "the change is reverted"}, ""},
} {
_, err := UnfinishedTools(unfinished, c.acts)[1].Run(c.args)
_, held := unfinished["main.app"]
if c.refusal != "" && (err == nil || !strings.Contains(err.Error(), c.refusal) || !held) {
t.Errorf("runner_unfinished_drop %s: %v, the key held %t", c.name, err, held)
}
if c.refusal == "" && (err != nil || held) {
t.Errorf("runner_unfinished_drop %s: %v, the key held %t", c.name, err, held)
}
}
}
// unfinishedOfMem is a state unfinished in memory.
type unfinishedOfMem map[string]string
func (u unfinishedOfMem) Keys() ([]string, error) { return queueOf(u).Keys() }
func (u unfinishedOfMem) Value(k string) (json.RawMessage, error) {
if v, ok := u[k]; ok {
return json.RawMessage(v), nil
}
return nil, nil
}
func (u unfinishedOfMem) Delete(k string) error { delete(u, k); return nil }
type recordingActs struct{}
func (recordingActs) RecordHandAct(views.HandAct) error { return nil }
// queueOf is a state queue in memory, its keys in no order.
type queueOf map[string]string
@@ -0,0 +1,47 @@
package main
import (
"fmt"
"strings"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-pipeline/internal/pipeline"
)
// ShowTools are runner_show and runner_movers, a pipeline read in parts (design 47 H2 §789): show's filters applied while
// reading, at most 768 KB with capped, total and the next page's cursor; and one word per step of each mover.
func ShowTools(r pipeline.Reads) []stdio.Tool {
return []stdio.Tool{
{Name: Prefix + "_show",
Description: "One pipeline in parts: its record with the steps the filters hold (a mover, a step kind, a step's " +
"outcome), at most 768 KB with capped, the filters' total and next, the cursor of the next page.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"pipeline": map[string]any{"type": "string", "description": "the pipeline's key"},
"mover": map[string]any{"type": "string", "description": "only this mover's steps"},
"step": map[string]any{"type": "string", "description": "only the steps of this kind (build, lint, unit, integration, validate, composition, review, approval)"},
"state": map[string]any{"type": "string", "description": "only the steps whose outcome is this (passed, failed, running, could not run, ...)"},
"next": map[string]any{"type": "string", "description": "the cursor the previous page answered"},
}, "required": []string{"pipeline"}},
Run: func(a map[string]any) (any, error) {
in, err := as[seat.ShowInput](a)
if err != nil {
return nil, err
}
return r.Show(in)
}},
{Name: Prefix + "_movers",
Description: "Each mover of one pipeline with one word per step: passed, failed, running, waiting or not-run.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"pipeline": map[string]any{"type": "string", "description": "the pipeline's key"},
}, "required": []string{"pipeline"}},
Run: func(a map[string]any) (any, error) {
key, _ := a["pipeline"].(string)
if strings.TrimSpace(key) == "" {
return nil, fmt.Errorf("name the pipeline's key")
}
return r.Movers(key)
}},
}
}
@@ -0,0 +1,105 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-pipeline/internal/pipeline"
)
// UnfinishedState is the runner's state unfinished as the tools read and drop it.
type UnfinishedState interface {
pipeline.Kept
Delete(key string) error
}
// keptDelete is a state of the runner read and deleted through the runtime.
type keptDelete struct{ kept }
func (k keptDelete) Delete(key string) error { return k.s.Delete(key) }
// Unfinished is one mover kept in the state unfinished, as runner_unfinished lists it.
type Unfinished struct {
Key string `json:"key"`
pipeline.UnfinishedEntry
}
// UnfinishedTools are runner_unfinished and runner_unfinished_drop over the runner's state unfinished (design 47 H2 §786):
// every mover a pipeline's end left unfinished, and one key dropped by hand, with why, written in the hand-act log first.
// runner_unfinished_place, which puts one in a pipeline of its own, asks the seat verb place-unfinished (VerbTools).
func UnfinishedTools(st UnfinishedState, acts views.HandActs) []stdio.Tool {
str := map[string]any{"type": "string"}
return []stdio.Tool{
{Name: Prefix + "_unfinished",
Description: "The movers a pipeline's end left unfinished (design 47 H2 §786): each key <branch>.<module> with its fingerprint, the catalog revision and the pipeline it was left in, and that pipeline's outcome. Each is put in the pipeline of its branch's next catalog revision; runner_unfinished_place puts one in a pipeline of its own now, runner_unfinished_drop drops one.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return unfinishedOf(st) }},
{Name: Prefix + "_unfinished_drop",
Description: "Drop one mover from the state unfinished by hand (design 47 H2 §786): its key <branch>.<module> and why, written in the hand-act log first; the mover is then put in no pipeline again until its fingerprint moves.",
Input: map[string]any{"type": "object", "properties": map[string]any{"key": str, "why": str}, "required": []string{"key", "why"}},
Run: func(args map[string]any) (any, error) {
in, err := as[struct {
Key string `json:"key"`
Why string `json:"why"`
}](args)
if err != nil {
return nil, err
}
return dropUnfinished(st, acts, in.Key, in.Why)
}},
}
}
func unfinishedOf(st pipeline.Kept) ([]Unfinished, error) {
keys, err := st.Keys()
if err != nil {
return nil, fmt.Errorf("the state unfinished is not read: %w", err)
}
slices.Sort(keys)
out := []Unfinished{}
for _, k := range keys {
raw, err := st.Value(k)
if err != nil || raw == nil {
continue
}
u := Unfinished{Key: k}
if err := json.Unmarshal(raw, &u.UnfinishedEntry); err != nil {
return nil, fmt.Errorf("the unfinished entry %s is not read: %w", k, err)
}
out = append(out, u)
}
return out, nil
}
func dropUnfinished(st UnfinishedState, acts views.HandActs, key, why string) (any, error) {
if strings.TrimSpace(key) == "" {
return nil, errors.New("runner_unfinished_drop names the unfinished entry's key")
}
if msg := whyOf(why); msg != "" {
return nil, errors.New(msg)
}
raw, err := st.Value(key)
if err != nil {
return nil, fmt.Errorf("the unfinished entry %s is not read: %w", key, err)
}
if raw == nil {
return nil, fmt.Errorf("the state unfinished holds no %s: nothing was dropped", key)
}
if acts == nil {
return nil, errors.New("no way to record a hand act is given: nothing was dropped")
}
if err := acts.RecordHandAct(views.HandAct{What: "dropped the unfinished mover " + key + " by hand", Why: strings.TrimSpace(why),
Cause: "unfinished-drop", Verb: Prefix + "_unfinished_drop"}); err != nil {
return nil, fmt.Errorf("the hand act is not recorded, so nothing was dropped: %w", err)
}
if err := st.Delete(key); err != nil {
return nil, fmt.Errorf("the unfinished entry %s is not deleted: %w", key, err)
}
return map[string]any{"dropped": key}, nil
}
@@ -9,8 +9,8 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
)
// VerbTools are runner_start, runner_run, runner_accept, runner_place and runner_ask_again: the runner's seat verbs start,
// run, accept, place (§748) and ask-again, reachable as tools (design 47 G §83, §482), as runner_rerun reaches rerun. Each asks its verb over the bus with
// VerbTools are runner_start, runner_run, runner_accept, runner_place, runner_unfinished_place and runner_ask_again: the
// runner's seat verbs start, run, accept, place (§748), place-unfinished (H2 §786) and ask-again, reachable as tools (design 47 G §83, §482), as runner_rerun reaches rerun. Each asks its verb over the bus with
// the arguments it was given and answers the verb's answer; the verb records the hand act and refuses what it refuses.
func VerbTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool {
str := map[string]any{"type": "string"}
@@ -63,6 +63,15 @@ func VerbTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.
}
return whyOf(in.Why)
}),
verbTool[seat.PlaceUnfinishedInput](ask, "unfinished_place", seat.VerbPlaceUnfinished,
"Put a mover kept in the state unfinished in a pipeline of its own at once, by the seat verb place-unfinished (design 47 H2 §786): the entry's key <branch>.<module> (runner_unfinished lists them) and why; written in the hand-act log. The answer names the pipeline, keyed <branch>.<revision>.unfinished.<module>.",
obj([]string{"key", "why"}, map[string]any{"key": str, "why": str}),
func(in seat.PlaceUnfinishedInput) string {
if in.Key == "" {
return "place-unfinished names the unfinished entry's key"
}
return whyOf(in.Why)
}),
verbTool[seat.AskAgainInput](ask, "ask_again", seat.VerbAskAgain,
"Ask a declined approval step of a pipeline once more by the seat verb ask-again: the pipeline's key, the mover's module (needed when the pipeline moves several) and why.",
obj([]string{"pipeline", "why"}, map[string]any{"pipeline": str, "module": str, "why": str}),
@@ -36,6 +36,10 @@ func TestVerbTools(t *testing.T) {
{"end-again", "runner_end_again", "seat:mesh-pipeline-runner.end-again", "", map[string]any{"pipeline": "main.20", "why": "left running"},
seat.EndAgainInput{Pipeline: "main.20", Why: "left running"}},
{"end-again no why", "runner_end_again", "", "names why", map[string]any{"pipeline": "main.20"}, nil},
// design 47 H2 §786, H-A-398: runner_unfinished_place asks the verb place-unfinished with the entry's key and why.
{"unfinished place", "runner_unfinished_place", "seat:mesh-pipeline-runner.place-unfinished", "", map[string]any{"key": "main.app", "why": "now"},
seat.PlaceUnfinishedInput{Key: "main.app", Why: "now"}},
{"unfinished place no key", "runner_unfinished_place", "", "entry's key", map[string]any{"why": "now"}, nil},
{"ask-again no why", "runner_ask_again", "", "names why", map[string]any{"pipeline": "p"}, nil},
} {
t.Run(c.name, func(t *testing.T) {
@@ -20,6 +20,7 @@ import (
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -73,6 +74,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// The movers a pipeline's end left unfinished, put in a pipeline again at their branch's next revision (design 47 H2 §786).
unfinishedKV, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, pipeline.StateUnfinished), "put a mover left unfinished in a pipeline again")
if err != nil {
return err
}
unfinished := &pipeline.Unfinished{Store: kvUnfinished{ctx, unfinishedKV}, Logf: logger.Printf}
// The toolchain of each language as heard agreed, for the steps' asks (§416, A-104).
toolchains, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "toolchains"), "name a step's toolchain")
if err != nil {
@@ -160,13 +167,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// design 47 H2 §790, §781: the stalled bound, read as the tools program reads it; a value that is no duration stops the start.
stalledBound, err := stdio.SettingDuration(pipeline.StalledBoundSetting, pipeline.DefaultStalledBound)
if err != nil {
return err
}
logger.Printf("health: a running pipeline with nothing open and nothing newly placed, or a step whose ask is untaken, is named after %s (%s)", stalledBound, pipeline.StalledBoundSetting)
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
// for its step.
starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say,
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, publish: js.PublishMsg, cred: cred, bound: askBound, logf: logger.Printf}, Name: cred.Module,
Registration: rv.Registrations.Registration, RegistrationsRead: func() bool { return chanClosed(rv.Registrations.Ready()) }, Steps: busStepAsker{ctx: ctx, publish: js.PublishMsg, cred: cred, bound: askBound, logf: logger.Printf}, Name: cred.Module,
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing,
Findings: rv.Findings, Observer: cred.Module} // a mover waiting on its registration is registration-missing (H-A-262)
Findings: rv.Findings, Observer: cred.Module, // a mover waiting on its registration is registration-missing (H-A-262)
Unfinished: unfinished}
hearer.Unfinished = unfinished
hearer.AskSteps = starter.AskSteps
hearer.RegistrationWaits = func() { starter.JudgeRegistrationMissing(store) } // at every end (H-A-265 (4))
logger.Printf("asks: an ask's bound %s, counted from its put until its holder's ask-taken, then from the take and each step-progress; past it the step ends could not run and its ask is dropped", askBound)
@@ -389,3 +404,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// ownBucket is a state of the runner's old state section on the bus, `<module>_<state>`.
func ownBucket(module, state string) string { return module + "_" + state }
// chanClosed says whether c is closed now, without waiting.
func chanClosed(c <-chan struct{}) bool {
select {
case <-c:
return true
default:
return false
}
}
@@ -54,7 +54,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", seat.StateQueue} {
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", seat.StateQueue, pipeline.StateUnfinished} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
@@ -304,6 +304,16 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
t.Fatal(err)
}
outcome("main.4", seat.PipelineOutcomeCancelled)
// design 47 H2 §786: the cancel leaves main.4's lib unfinished; dropped here, as runner_unfinished_drop does, so the
// next take-ins move only what they name.
unfinished, err := js.KeyValue(ctx, module+"_"+pipeline.StateUnfinished)
if err != nil {
t.Fatal(err)
}
waitFor(t, ctx, logs, func() bool { _, err := unfinished.Get(ctx, "main.lib"); return err == nil })
if err := unfinished.Delete(ctx, "main.lib"); err != nil {
t.Fatal(err)
}
outcome("main.6", seat.PipelineOutcomeRunning)
waitFor(t, ctx, logs, func() bool { _, ok := finding(pipeline.QueueFullKind); return !ok })
if p := get("main.7"); p.Outcome != seat.PipelineOutcomeWaiting {
@@ -22,6 +22,7 @@ import (
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -47,124 +48,23 @@ func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return
// of the build, and ends the placed review step when a session calls `review`, answered by the process with the caller the
// bus proved; the pipeline then ends passed.
func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) {
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
}
// bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl.
for _, state := range []string{"bounds", TestRunsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
}
}
// Kept pipelines in the old form, their steps inline in the record, as the runner's start finds them on a live bus
// (main.41-46, one of 437 steps): each start path reads them at once from its own goroutine (H-A-187, H §774).
keptPipelines(t, ctx, js, module)
// The step log, the runner's old logs section (§470): the stream the bus makes for it.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
t.Fatal(err)
}
findingsStream(t, ctx, js, module)
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
if err != nil {
t.Fatal(err)
}
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
asks, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy})
if err != nil {
t.Fatal(err)
}
// The seats the runner calls, answered as their holders do.
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional", Validate: "bypass", Composition: "bypass", Approval: "bypass"}}}
answer := func(subject string, body any) {
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": body})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
}
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: []catalog.Registration{reg}})
// design 47 H2 §790: the stalled bound is the setting stalled-bound, here one minute, read as the node-engine composes it.
t.Setenv(stdio.SettingWord(pipeline.StalledBoundSetting), "1m")
// The branch main has no revision yet at the runner's start (none-yet), then revision 1 holding app (A-336).
var revised atomic.Bool
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) {
var a catalog.RevisionAnswer
w := startWhole(t, ctx, []catalog.Registration{reg}, func([]byte) catalog.RevisionAnswer {
if revised.Load() {
a = catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 3},
return catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 3},
Entries: []catalog.CatalogEntry{{Module: "app", Commit: "c1", Fp: "src2:aa"}}}
} else {
a.NoneYet = true
}
raw, _ := json.Marshal(map[string]any{"result": a})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
// The seat list (A-186): the seat mesh-catalog's seat-definitions names the runner's seat, whose definition its write
// verbs' callers are checked against, as the contracts' file holds it.
def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json")
if err != nil {
t.Fatal(err)
}
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) {
var in struct {
Seat string `json:"seat"`
}
_ = json.Unmarshal(m.Data, &in)
entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"}
var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}}
if in.Seat != "" {
entry.Definition = def
result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1}
}
raw, _ := json.Marshal(map[string]any{"result": result})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
return catalog.RevisionAnswer{NoneYet: true}
})
const module = "mesh-delivery-pipeline"
nc, js, asks, pipelines, logs := w.nc, w.js, w.asks, w.pipelines, w.logs
dir := t.TempDir()
broker := filepath.Join(dir, "broker")
raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"})
if err := os.WriteFile(broker, raw, 0o600); err != nil {
t.Fatal(err)
}
logs := &syncBuffer{}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() {
done <- run(runCtx, func(k string) string {
if k == process.BrokerFileEnv {
return broker
}
return ""
}, logs)
}()
defer func() { stop(); <-done }()
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
if err != nil {
t.Fatal(err)
}
// A take-in.taken names one entry of the source revision; the runner places the pipeline of the registered module.
taken, _ := json.Marshal(catalog.TakeInTaken{Branch: "main", Source: 3, Call: "0123456789abcdef",
Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}})
@@ -180,6 +80,15 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
if len(p.Steps) != 0 {
t.Fatalf("a take-in before its branch's half placed %d steps, want none (A-336)", len(p.Steps))
}
// design 47 H2 §790: at one minute the health names the pipeline with nothing placed, a minute after its start.
bound, err := stdio.SettingDuration(pipeline.StalledBoundSetting, pipeline.DefaultStalledBound)
if err != nil || bound != time.Minute || !strings.Contains(logs.String(), "is named after 1m0s (stalled-bound)") {
t.Fatalf("the stalled bound %s (%v); the process says it: %t", bound, err, strings.Contains(logs.String(), "is named after 1m0s"))
}
noConsumers := kvKept{ctx, mustKV(t, ctx, js, module+"_consumers")}
if a := pipeline.Health(kvKept{ctx, pipelines}, noConsumers, p.Started.Add(61*time.Second), bound); a.Healthy || !strings.Contains(a.Why, key+" (nothing open") {
t.Fatalf("a minute after its start, with nothing placed, the health says %+v", a)
}
// The revision said is main's first half: held, it places the pipeline's steps from the registration (A-327).
revised.Store(true)
rev, _ := json.Marshal(catalog.CatalogRevisionEvent{V: 1, Revision: 1, Kind: "trunk", Branch: "main", Source: 3})
@@ -191,6 +100,10 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
if build.Outcome != seat.StepResultOutcomeRunning || build.Ask == "" {
t.Fatalf("the build step = %+v", build)
}
// Once a step is placed in it, the health no longer names it (§790).
if a := pipeline.Health(kvKept{ctx, pipelines}, noConsumers, p.Started.Add(61*time.Second), bound); strings.Contains(a.Why, key+" (nothing open") {
t.Fatalf("with its build placed, the health still names %s: %+v", key, a)
}
// The build is asked of the seat mesh-builds: its ask is on its work queue.
waitFor(t, ctx, logs, func() bool { info, err := asks.Info(ctx); return err == nil && info.State.Msgs >= 1 })
@@ -270,11 +183,142 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
e, err := consumers.Get(ctx, "n1_"+module)
return err == nil && json.Unmarshal(e.Value(), &facts) == nil && facts.OldestUnacknowledged.IsZero() && facts.Acknowledged > 0
})
if a := pipeline.Health(kvKept{ctx, pipelines}, kvKept{ctx, consumers}, time.Now()); !a.Healthy {
if a := pipeline.Health(kvKept{ctx, pipelines}, kvKept{ctx, consumers}, time.Now(), bound); !a.Healthy {
t.Errorf("runner_health over the bus: %+v", a)
}
}
// whole is the runner started whole as its process is, against this test's bus (design 47 G §22, §54): its states, streams
// and seat list made as the bus makes them from its manifest, the catalog's registrations and revisions answered by regs and
// revision, its journal in logs.
type whole struct {
nc *nats.Conn
js jetstream.JetStream
asks jetstream.Stream
pipelines jetstream.KeyValue
logs *syncBuffer
}
func startWhole(t *testing.T, ctx context.Context, regs []catalog.Registration, revision func(in []byte) catalog.RevisionAnswer) whole {
t.Helper()
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue", pipeline.StateUnfinished} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
}
// bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl.
for _, state := range []string{"bounds", TestRunsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
}
}
// Kept pipelines in the old form, their steps inline in the record, as the runner's start finds them on a live bus
// (main.41-46, one of 437 steps): each start path reads them at once from its own goroutine (H-A-187, H §774).
keptPipelines(t, ctx, js, module)
// The step log, the runner's old logs section (§470): the stream the bus makes for it.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
t.Fatal(err)
}
findingsStream(t, ctx, js, module)
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
if err != nil {
t.Fatal(err)
}
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
asks, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy})
if err != nil {
t.Fatal(err)
}
// The seats the runner calls, answered as their holders do.
answer := func(subject string, body any) {
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": body})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
}
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs})
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": revision(m.Data)})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
// The seat list (A-186): the seat mesh-catalog's seat-definitions names the runner's seat, whose definition its write
// verbs' callers are checked against, as the contracts' file holds it.
def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json")
if err != nil {
t.Fatal(err)
}
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) {
var in struct {
Seat string `json:"seat"`
}
_ = json.Unmarshal(m.Data, &in)
entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"}
var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}}
if in.Seat != "" {
entry.Definition = def
result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1}
}
raw, _ := json.Marshal(map[string]any{"result": result})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
dir := t.TempDir()
broker := filepath.Join(dir, "broker")
raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"})
if err := os.WriteFile(broker, raw, 0o600); err != nil {
t.Fatal(err)
}
logs := &syncBuffer{}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() {
done <- run(runCtx, func(k string) string {
if k == process.BrokerFileEnv {
return broker
}
return ""
}, logs)
}()
t.Cleanup(func() { stop(); <-done })
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
if err != nil {
t.Fatal(err)
}
return whole{nc: nc, js: js, asks: asks, pipelines: pipelines, logs: logs}
}
// mustKV is the bucket name.
func mustKV(t *testing.T, ctx context.Context, js jetstream.JetStream, name string) jetstream.KeyValue {
t.Helper()
kv, err := js.KeyValue(ctx, name)
if err != nil {
t.Fatal(err)
}
return kv
}
// kvKept reads a bucket as the tools bundle's states do.
type kvKept struct {
ctx context.Context
@@ -58,7 +58,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue"} {
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue", pipeline.StateUnfinished} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
@@ -0,0 +1,49 @@
package main
import (
"context"
"errors"
"github.com/nats-io/nats.go/jetstream"
)
// kvUnfinished is the runner's state unfinished on the bus (design 47 H2 §786), as pipeline.UnfinishedStore reads it: a key
// it does not hold is nil.
type kvUnfinished struct {
ctx context.Context
kv jetstream.KeyValue
}
func (u kvUnfinished) Get(key string) ([]byte, error) {
e, err := u.kv.Get(u.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return e.Value(), nil
}
func (u kvUnfinished) Put(key string, value []byte) error {
_, err := u.kv.Put(u.ctx, key, value)
return err
}
func (u kvUnfinished) Delete(key string) error { return u.kv.Delete(u.ctx, key) }
// Keys are the keys matching filter, `*` one key part.
func (u kvUnfinished) Keys(filter string) ([]string, error) {
lister, err := u.kv.ListKeysFiltered(u.ctx, filter)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
if err != nil {
return nil, err
}
var out []string
for k := range lister.Keys() {
out = append(out, k)
}
return out, nil
}
@@ -0,0 +1,143 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"slices"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"mesh-delivery-pipeline/internal/pipeline"
)
// design 47 H2 §786, whole: of a pipeline of two movers one fails its unit step; pipeline-ended names the other in
// unfinished and the runner keeps it in its state unfinished; the next take-in's pipeline carries it, its composition is
// asked and passes (it is wanted), and its key goes once that pipeline ends passed.
func TestAMoverLeftUnfinishedIsPutInTheNextPipelineAndWanted(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
steps := catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "bypass", Validate: "required", Composition: "required",
Review: "bypass", Approval: "bypass"}
regs := []catalog.Registration{{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: steps}},
{Module: "lib", Branches: []string{"main"}, Origin: "o/lib", Policy: catalog.Policy{Steps: steps}}}
var newest atomic.Int64
newest.Store(1)
w := startWhole(t, ctx, regs, func(in []byte) catalog.RevisionAnswer {
var q catalog.RevisionInput
_ = json.Unmarshal(in, &q)
n := q.Revision
if n == 0 {
n = newest.Load()
}
lib := "l1"
if n == 2 {
lib = "l2"
}
return catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: n, Kind: "trunk", Branch: "main", Source: n + 2},
Entries: []catalog.CatalogEntry{{Module: "app", Commit: "c1", Fp: "src2:aa"},
{Module: "lib", Commit: lib, Fp: "src2:" + lib, Artifacts: []catalog.Artifact{{Ref: "lib:lib", Language: "go", Kind: "program"}}},
{Module: "mesh-build-go", Commit: "c0", Fp: "src2:go", Acts: []catalog.ActingRule{{Step: "build", On: "go", Seat: nodebuildgo.Seat}}}}}
})
ended := make(chan seat.PipelineEnded, 4)
if _, err := w.nc.Subscribe("mesh.seat.mesh-pipeline-runner.event.pipeline-ended.>", func(m *nats.Msg) {
var pe seat.PipelineEnded
if json.Unmarshal(m.Data, &pe) == nil {
ended <- pe
}
}); err != nil {
t.Fatal(err)
}
unfinished, err := w.js.KeyValue(ctx, "mesh-delivery-pipeline_"+pipeline.StateUnfinished)
if err != nil {
t.Fatal(err)
}
takeIn := func(source, revision int64, entries ...catalog.TakeInEntry) {
raw, _ := json.Marshal(catalog.TakeInTaken{Branch: "main", Source: source, Revision: revision, Call: "0123456789abcdef", Entries: entries})
if _, err := w.js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", raw); err != nil {
t.Fatal(err)
}
}
// answer ends every running step of key once, as its seat's holder says it: lib's unit fails when fail says so.
answered := map[string]bool{}
answer := func(key string, fail bool) func() bool {
return func() bool {
var p seat.Pipeline
if !readPipeline(ctx, w.pipelines, key, &p) {
return false
}
for _, s := range p.Steps {
open := s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode
if !open || s.Ask == "" || answered[s.Ask+s.Module] {
continue
}
answered[s.Ask+s.Module] = true
m, _ := pipeline.MoverOf(p, s.Module)
outcome := "passed"
if fail && s.Module == "lib" && s.Step == seat.StepResultStepUnit {
outcome = "failed"
}
body := map[string]any{"v": 1, "id": s.Ask, "pipeline": key, "step": s.Step, "module": s.Module, "commit": m.Commit,
"fingerprint": m.Fingerprint, "node": "n1", "outcome": outcome, "artifact": s.Artifact}
if s.Seat == builds.Seat {
body["entries"] = []builds.BuildEntry{{Module: s.Module, Commit: m.Commit, Fingerprint: m.Fingerprint}}
}
raw, _ := json.Marshal(body)
if _, err := w.js.Publish(ctx, "mesh.seat."+s.Seat+".event.step-ended.n1", raw); err != nil {
t.Fatal(err)
}
}
return p.Outcome != seat.PipelineOutcomeRunning
}
}
endOf := func() seat.PipelineEnded {
select {
case pe := <-ended:
return pe
case <-ctx.Done():
t.Fatalf("no pipeline-ended was said; the runner said:\n%s", w.logs.String())
}
return seat.PipelineEnded{}
}
takeIn(3, 1, catalog.TakeInEntry{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, catalog.TakeInEntry{Module: "lib", Commit: "l1", Fingerprint: "src2:l1"})
waitFor(t, ctx, w.logs, answer("main.1", true))
if pe := endOf(); pe.Pipeline != "main.1" || pe.Outcome != seat.PipelineEndedOutcomeFailed || !slices.Equal(pe.Unfinished, []string{"app"}) {
t.Fatalf("pipeline-ended = %s %s unfinished %v, want main.1 failed leaving app unfinished\n%s", pe.Pipeline, pe.Outcome, pe.Unfinished, w.logs.String())
}
if e, err := unfinished.Get(ctx, "main.app"); err != nil || !json.Valid(e.Value()) {
t.Fatalf("the state unfinished holds no main.app: %v", err)
}
// The next take-in moves lib alone, at a new fingerprint: its pipeline carries app, whose composition is asked and passes.
newest.Store(2)
rev, _ := json.Marshal(catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4, CallKind: catalog.CatalogRevisionEventCallKindTakeIn})
if _, err := w.js.Publish(ctx, catalog.SubjectCatalogRevision("main"), rev); err != nil {
t.Fatal(err)
}
takeIn(4, 2, catalog.TakeInEntry{Module: "lib", Commit: "l2", Fingerprint: "src2:l2"})
waitFor(t, ctx, w.logs, answer("main.2", false))
var p seat.Pipeline
if !readPipeline(ctx, w.pipelines, "main.2", &p) || !pipeline.Moves(p, "app") {
t.Fatalf("main.2 moves %+v, want app carried", p.Movers)
}
composed := false
for _, s := range p.Steps {
composed = composed || (s.Module == "app" && s.Step == seat.StepResultStepComposition && s.Outcome == seat.StepResultOutcomePassed && s.Ask != "")
}
if pe := endOf(); pe.Pipeline != "main.2" || pe.Outcome != seat.PipelineEndedOutcomePassed || !composed || len(pe.Unfinished) != 0 {
t.Fatalf("main.2 ended %s, app composed %t, unfinished %v\n%s", pe.Outcome, composed, pe.Unfinished, w.logs.String())
}
if _, err := unfinished.Get(ctx, "main.app"); err == nil {
t.Error("main.app is still unfinished after main.2 passed")
}
}
@@ -38,19 +38,21 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu
return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v))
}
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, end-again, start, and ask-again; and
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, end-again, place,
// place-unfinished (design 47 H2 §786), start, and ask-again; and
// the test runs' verbs test-runs and test-runs-drop (A-469g, A-470i).
func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }),
seat.VerbReview: verbOf(func(c string, in seat.ReviewInput) (any, error) { return v.Review(c, in) }),
seat.VerbRerun: verbOf(func(c string, in seat.RerunInput) (any, error) { return v.Rerun(c, in) }),
seat.VerbAccept: verbOf(func(c string, in seat.AcceptInput) (any, error) { return v.Accept(c, in) }),
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
seat.VerbEndAgain: verbOf(func(c string, in seat.EndAgainInput) (any, error) { return v.EndAgain(c, in) }),
seat.VerbPlace: verbOf(func(c string, in seat.PlaceInput) (any, error) { return v.Place(c, in) }),
seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }),
seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }),
seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }),
seat.VerbReview: verbOf(func(c string, in seat.ReviewInput) (any, error) { return v.Review(c, in) }),
seat.VerbRerun: verbOf(func(c string, in seat.RerunInput) (any, error) { return v.Rerun(c, in) }),
seat.VerbAccept: verbOf(func(c string, in seat.AcceptInput) (any, error) { return v.Accept(c, in) }),
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
seat.VerbEndAgain: verbOf(func(c string, in seat.EndAgainInput) (any, error) { return v.EndAgain(c, in) }),
seat.VerbPlace: verbOf(func(c string, in seat.PlaceInput) (any, error) { return v.Place(c, in) }),
seat.VerbPlaceUnfinished: verbOf(func(c string, in seat.PlaceUnfinishedInput) (any, error) { return v.PlaceUnfinished(c, in) }),
seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }),
seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }),
// §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs.
seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }),
// A-470i: test-runs too, since the process holds test-runs-cap-mib, the cap its fill is said against.
@@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) {
got = append(got, v)
}
slices.Sort(got)
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbEndAgain, seat.VerbPlace, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop}
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbEndAgain, seat.VerbPlace, seat.VerbPlaceUnfinished, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop}
slices.Sort(want)
if !slices.Equal(got, want) {
t.Errorf("the process answers %v, want %v", got, want)
@@ -7,9 +7,11 @@ import (
"strings"
"time"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
)
// StateConsumers is the runner's old state section of its durable consumers' facts, a key per consumer, written by
@@ -20,32 +22,39 @@ const StateConsumers = process.ConsumersState
// behind (§76).
const BehindAfter = 5 * time.Minute
// StalledAfter is how long a running pipeline may have nothing open and nothing newly placed before the runner's health
// names it stalled (H-A-311 (3)): the mesh's own rule for nothing moving (nothing-moved-in-five-minutes), not a setting
// (H-A-322).
const StalledAfter = 5 * time.Minute
// DefaultStalledBound is the setting stalled-bound's default (design 47 H2 §790): how long a running pipeline may have
// nothing open and nothing newly placed, or a step's ask stay untaken (§778), before the runner's health names it; the
// mesh's own rule for nothing moving (nothing-moved-in-five-minutes).
const DefaultStalledBound = 5 * time.Minute
// StalledBoundSetting is the runner's setting of that bound, read by its process and its tools program (§790, §781).
const StalledBoundSetting = "stalled-bound"
// Stalled is the key of each running pipeline, as show reads it, that has had nothing open (no step asked and not ended,
// no required step waiting but on a package's publish) and nothing newly placed, started or ended for StalledAfter at now (H-A-311 (3)), each with
// how long, in key order.
func Stalled(pipelines Kept, now time.Time) ([]string, error) {
// no required step waiting but on a package's publish) and nothing newly placed, started or ended for bound at now
// (H-A-311 (3), design 47 H2 §790), each with how long, in key order; and each step of a running pipeline whose ask is
// still untaken bound after it was put (§778), each with its seat and how long.
func Stalled(pipelines Kept, now time.Time, bound time.Duration) ([]string, error) {
all, err := Reads{Pipelines: pipelines}.pipelines()
if err != nil {
return nil, err
}
var out []string
var out, untaken []string
for _, p := range all {
if p.Outcome != seat.PipelineOutcomeRunning {
continue
}
last, busy := startedOf(p), false
for _, st := range p.Steps {
if st.Ask != "" && st.Ended == nil && st.TakenAt == nil && st.AskedAt != nil && open(st) && now.Sub(*st.AskedAt) >= bound {
untaken = append(untaken, fmt.Sprintf("%s %s %s (its ask on %s untaken for %s)", p.Pipeline, st.Module, st.Step,
st.Seat, now.Sub(*st.AskedAt).Round(time.Second)))
}
// A required step waiting (an approval, a composition on its validate) is open, as the end's judgement holds it;
// an optional one waiting holds nothing (A-104), and neither does one waiting on a package's publish, as a mover
// with no steps waiting on one does not (H-A-326 (2)).
if open(st) || (st.Outcome == seat.StepResultOutcomeWaiting && st.Status == seat.StepResultStatusRequired && !pinHeld(st)) {
busy = true
break
}
for _, at := range []*time.Time{st.Started, st.Ended} {
if at != nil && at.After(last) {
@@ -53,42 +62,42 @@ func Stalled(pipelines Kept, now time.Time) ([]string, error) {
}
}
}
if busy || last.IsZero() || now.Sub(last) < StalledAfter {
if busy || last.IsZero() || now.Sub(last) < bound {
continue
}
out = append(out, fmt.Sprintf("%s (nothing open and nothing placed for %s)", p.Pipeline, now.Sub(last).Round(time.Second)))
}
return out, nil
return append(out, untaken...), nil
}
// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state
// pipelines answers, as show reads it, each durable consumer's oldest unacknowledged message waits less than
// BehindAfter, and no running pipeline is stalled (H-A-311 (3)); otherwise unhealthy naming each consumer behind and
// how long, and each stalled pipeline by its key.
func Health(pipelines, consumers Kept, now time.Time) health.Answer {
a, _ := HealthStalled(pipelines, consumers, now)
func Health(pipelines, consumers Kept, now time.Time, bound time.Duration) health.Answer {
a, _ := HealthStalled(pipelines, consumers, now, bound)
return a
}
// HealthStalled is Health with the keys of the running pipelines it names stalled, so the tools program says each in its
// journal once (H-A-313).
func HealthStalled(pipelines, consumers Kept, now time.Time) (health.Answer, []string) {
func HealthStalled(pipelines, consumers Kept, now time.Time, bound time.Duration) (health.Answer, []string) {
if _, err := pipelines.Keys(); err != nil {
return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err)), nil
}
stalled, err := Stalled(pipelines, now)
stalled, err := Stalled(pipelines, now, bound)
if err != nil {
return health.Unhealthy(fmt.Sprintf("The runner's running pipelines cannot be read as show reads them (%v).", err)), nil
}
var keys []string
for _, s := range stalled {
keys = append(keys, strings.SplitN(s, " ", 2)[0])
keys = append(keys, strings.SplitN(s, " (", 2)[0])
}
a := consumersHealth(consumers, now)
if len(stalled) == 0 {
return a, nil
}
why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed for %s or more: %s.", StalledAfter, strings.Join(stalled, ", "))
why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed, or a step whose ask is untaken, for %s or more: %s.", bound, strings.Join(stalled, ", "))
if !a.Healthy {
why = a.Why + " " + why
}
@@ -128,3 +137,35 @@ func consumersHealth(consumers Kept, now time.Time) health.Answer {
}
return health.Healthy(fmt.Sprintf("The runner answers show, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter))
}
// RegistrationsRead is runner_health's registrations-read (design 47 H2 §788): whether the runner's registrations view's
// first read is in, and since when; before it, since when it was asked.
type RegistrationsRead struct {
Read bool `json:"read"`
Since *time.Time `json:"since,omitempty"`
}
// RegistrationsReadOf reads registrations-read from the runner's own first-views state, where its process keeps each first
// view's status (views.StatusState): read once the seat mesh-catalog's registrations view was answered.
func RegistrationsReadOf(firstViews Kept) (RegistrationsRead, error) {
raw, err := firstViews.Value(views.StatusKey(catalog.Seat, catalog.StateRegistrations))
if err != nil {
return RegistrationsRead{}, fmt.Errorf("the first view of the registrations is not read from %s: %w", views.StatusState, err)
}
if len(raw) == 0 {
return RegistrationsRead{}, nil
}
var st views.Status
if err := json.Unmarshal(raw, &st); err != nil {
return RegistrationsRead{}, fmt.Errorf("the first view of the registrations in %s: %w", views.StatusState, err)
}
if !st.Answered.IsZero() {
at := st.Answered
return RegistrationsRead{Read: true, Since: &at}, nil
}
if !st.Asked.IsZero() {
at := st.Asked
return RegistrationsRead{Since: &at}, nil
}
return RegistrationsRead{}, nil
}
@@ -65,8 +65,37 @@ func TestHealth(t *testing.T) {
{"a pipeline waiting only on packages is named", heldOnPublish(piped(now, "main.137", -20*time.Minute, seat.StepResultOutcomeWaiting), "main.137"), age("n_runner", 0), false,
"main.137 (nothing open and nothing placed for 20m0s)"},
}
// design 47 H2 §778, §790: a step whose ask is untaken past the bound is named with its seat; the bound is the setting's.
untaken := func(asked time.Duration, taken bool) mapKept {
k := piped(now, "main.140", -time.Minute, seat.StepResultOutcomeWaitingForABuildNode)
var p seat.Pipeline
_ = json.Unmarshal(k["main.140"], &p)
at := now.Add(-asked)
p.Steps[0].Ask, p.Steps[0].Seat, p.Steps[0].AskedAt, p.Steps[0].Ended = "ask-1", "mesh-builds", &at, nil
if taken {
p.Steps[0].TakenAt = &at
}
raw, _ := json.Marshal(p)
k["main.140"] = raw
return k
}
rows = append(rows, []struct {
name string
pipelines Kept
consumers Kept
healthy bool
says string
}{
{"an ask untaken six minutes is named", untaken(6*time.Minute, false), age("n_runner", 0), false, "main.140 app build (its ask on mesh-builds untaken for 6m0s)"},
{"an ask untaken four minutes is not", untaken(4*time.Minute, false), age("n_runner", 0), true, ""},
{"a taken ask is not", untaken(20*time.Minute, true), age("n_runner", 0), true, ""},
}...)
if a := Health(piped(now, "main.137", -2*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), now, time.Minute); a.Healthy ||
!strings.Contains(a.Why, "main.137 (nothing open and nothing placed for 2m0s)") {
t.Errorf("at a one-minute bound a pipeline quiet two minutes: %q", a.Why)
}
for _, r := range rows {
a := Health(r.pipelines, r.consumers, now)
a := Health(r.pipelines, r.consumers, now, DefaultStalledBound)
if a.Healthy != r.healthy || !strings.Contains(a.Why, r.says) {
t.Errorf("%s: healthy %v %q; want %v saying %q", r.name, a.Healthy, a.Why, r.healthy, r.says)
}
@@ -2,6 +2,8 @@ package pipeline
import (
"errors"
"maps"
"slices"
"strings"
"testing"
"time"
@@ -87,14 +89,23 @@ func TestOnePipelineEndedNamesEveryMoverWithItsTier(t *testing.T) {
// R6/AC1 and AC2 (design 47 H §746; ADR 0682 §8, H-A-3): a mover's required step that ends `nothing to run` counts as
// passed, so the pipeline passes; one mover's required step failed fails the pipeline, naming that mover's step.
//
// design 47 H2 §786: the failed end names the other mover in pipeline-ended's unfinished and keeps it in the state
// unfinished; the failing mover's own key goes (it waits for its fingerprint to move), and a passed end deletes its movers'
// keys.
func TestThePipelinePassesOnNothingToRunAndFailsOnOneMover(t *testing.T) {
for _, c := range []struct {
name, lib, want string
unfinished []string
keys string
}{
{"nothing to run", seat.StepResultOutcomeNothingToRun, seat.PipelineOutcomePassed},
{"one mover failed", seat.StepResultOutcomeFailed, seat.PipelineOutcomeFailed},
{"nothing to run", seat.StepResultOutcomeNothingToRun, seat.PipelineOutcomePassed, nil, ""},
{"one mover failed", seat.StepResultOutcomeFailed, seat.PipelineOutcomeFailed, []string{"app"},
`main.app={"fingerprint":"src2:aa","revision":0,"pipeline":"main.5","outcome":"failed"}`},
} {
h, store, _, _ := twoMovers(t)
h, store, bodies, _ := twoMovers(t)
u := memUnfinished{"main.lib": `{"fingerprint":"src2:bb"}`, "main.app": `{"fingerprint":"src2:aa"}`}
h.Unfinished = &Unfinished{Store: u}
for _, ev := range []StepEnded{ended("a1", "lint", "passed"), libEnded(c.lib)} {
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ev); err != nil {
t.Fatal(err)
@@ -104,9 +115,44 @@ func TestThePipelinePassesOnNothingToRunAndFailsOnOneMover(t *testing.T) {
if p.Outcome != c.want || (c.want == seat.PipelineOutcomeFailed && !strings.Contains(p.Why, "lib lib:lib unit failed")) {
t.Errorf("%s: the pipeline ended %s (%s), want %s", c.name, p.Outcome, p.Why, c.want)
}
pe, _ := bodies["pipeline-ended.main.5"].(seat.PipelineEnded)
if !slices.Equal(pe.Unfinished, c.unfinished) || u.String() != c.keys {
t.Errorf("%s: pipeline-ended's unfinished %v and the state %q, want %v and %q", c.name, pe.Unfinished, u.String(), c.unfinished, c.keys)
}
}
}
// memUnfinished is a state unfinished in memory.
type memUnfinished map[string]string
func (m memUnfinished) Get(k string) ([]byte, error) {
if v, ok := m[k]; ok {
return []byte(v), nil
}
return nil, nil
}
func (m memUnfinished) Put(k string, v []byte) error { m[k] = string(v); return nil }
func (m memUnfinished) Delete(k string) error { delete(m, k); return nil }
func (m memUnfinished) Keys(filter string) ([]string, error) {
prefix := strings.TrimSuffix(filter, "*")
var out []string
for k := range m {
if strings.HasPrefix(k, prefix) {
out = append(out, k)
}
}
return out, nil
}
// String is the state's keys and values in key order.
func (m memUnfinished) String() string {
var out []string
for _, k := range slices.Sorted(maps.Keys(m)) {
out = append(out, k+"="+m[k])
}
return strings.Join(out, " ")
}
// R8 (design 47 H §746, H-A-52, H-A-62): a version-released writes its version into the mover of that module in the pipeline
// of the source revision built, on the branch it was released for, so pipeline-ended names it; another branch's pipeline,
// and a version heard again, write nothing.
@@ -29,7 +29,8 @@ func StepsKey(pipeline, mover string) string {
func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") }
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a
// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`) or a publish's (7 parts) and one mover.
// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`), an unfinished mover's own (6 parts, its fourth
// `unfinished`, H-A-398) or a publish's (7 parts) and one mover.
func IsStepsKey(key string) bool {
parts := strings.Split(key, ".")
if parts[0] != "steps" {
@@ -40,6 +41,8 @@ func IsStepsKey(key string) bool {
return true
case 5:
return parts[2] == runToken
case 6:
return parts[3] == unfinishedToken
}
return false
}
@@ -312,7 +315,7 @@ func (s *MoverStore) KeptRevisions(key string) (map[string]int64, error) {
}
// keptRevisionsField is the pipeline record's property kept-revisions, beside its movers (H-A-285): written and read by the
// runner alone, never part of the seat's Go form of a pipeline, which drops it on every decode.
// runner alone, never part of the seat's Go form of a pipeline, whose tolerant decode reads past it (design 47 H2 §780).
const keptRevisionsField = "kept-revisions"
// keptOf is the kept-revisions a record carries; none when it carries none.
@@ -332,20 +335,6 @@ func keptOf(raw []byte) (map[string]int64, error) {
return kept, nil
}
// withoutKept is record without its kept-revisions, the runner's own property and never part of the seat's Go form, so the
// seat's strict decode of the record the runner's reads make refuses no record carrying it (H-A-285, H-A-309).
func withoutKept(record []byte) ([]byte, error) {
var fields map[string]json.RawMessage
if err := json.Unmarshal(record, &fields); err != nil {
return nil, fmt.Errorf("the record is no object: %w", err)
}
if _, ok := fields[keptRevisionsField]; !ok {
return record, nil
}
delete(fields, keptRevisionsField)
return json.Marshal(fields)
}
// withKept is record with kept as its kept-revisions.
func withKept(record []byte, kept map[string]int64) ([]byte, error) {
var fields map[string]json.RawMessage
@@ -408,17 +408,18 @@ func keptOfStore(store any, key string) (map[string]int64, error) {
// keptEntries are the entries of branch's catalog revision rev, the one a same-source re-pin kept a mover at, read by its
// number (H-A-286): a kept mover is placed and pin-checked from them, never from the branch's held view; false while they
// cannot be read, and the mover waits.
func (s *Starter) keptEntries(branch string, rev int64) (map[string]catalog.CatalogEntry, bool) {
func (s *Starter) keptEntries(branch string, rev int64) (map[string]catalog.CatalogEntry, error) {
if s.Revision == nil {
s.logf("the runner reads no catalog revision, so a mover kept at revision %d of %s waits", rev, branch)
return nil, false
return nil, fmt.Errorf("the runner reads no catalog revision")
}
half, err := s.Revision(branch, rev)
if err != nil || half.Revision != rev {
s.logf("revision %d of %s, which a mover is kept at, is not read (%v): the mover waits for it (H-A-286)", rev, branch, err)
return nil, false
if err != nil {
return nil, err
}
return half.Entries, true
if half.Revision != rev {
return nil, fmt.Errorf("the view holds revision %d", half.Revision)
}
return half.Entries, nil
}
func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
@@ -3,9 +3,11 @@
package pipeline
import (
"encoding/base64"
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"time"
@@ -39,8 +41,12 @@ func (r Reads) fills() ([]seat.Fill, error) {
return f, nil
}
// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the
// pipeline's record with that mover's steps alone (H-A-187).
// MaxAnswer is the most a read answers (design 47 H2 §789, §811): 768 KB, the rest read from its cursor.
const MaxAnswer = 768 << 10
// Show is one pipeline, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the
// pipeline's record with that mover's steps alone (H-A-187). Its filters mover, step and state apply while it reads, and it
// answers at most MaxAnswer with capped, the filters' total and the cursor of the next page (design 47 H2 §789, §779).
func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
p, ok, err := r.pipeline(in.Pipeline)
if err != nil {
@@ -53,22 +59,130 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
if err != nil {
return seat.ShowAnswer{}, err
}
// Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers.
if m := strings.TrimSpace(in.Mover); m != "" {
if !Moves(p, m) {
return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m)
}
p.Steps = stepsOf(p, m)
from, err := pageFrom(in.Next)
if err != nil {
return seat.ShowAnswer{}, err
}
a := seat.ShowAnswer{Pipeline: p, Fill: fills}
// Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers.
m := strings.TrimSpace(in.Mover)
if m != "" && !Moves(p, m) {
return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m)
}
held := []seat.StepResult{}
for _, s := range p.Steps {
if s.Output != "" {
a.Outputs = append(a.Outputs, s.Output)
if (m == "" || s.Module == m) && (in.Step == "" || s.Step == in.Step) && (in.State == "" || s.Outcome == in.State) {
held = append(held, s)
}
}
p.Steps = []seat.StepResult{}
a := seat.ShowAnswer{Pipeline: p, Fill: fills, Total: int64(len(held))}
size := sizeOf(a)
for i := from; i < len(held); i++ {
n := sizeOf(held[i]) + 1
if s := held[i]; s.Output != "" {
n += len(s.Output) + 3
}
if size+n > MaxAnswer && len(a.Pipeline.Steps) > 0 {
a.Capped, a.Next = true, pageCursor(i)
break
}
size += n
a.Pipeline.Steps = append(a.Pipeline.Steps, held[i])
if held[i].Output != "" {
a.Outputs = append(a.Outputs, held[i].Output)
}
}
return a, nil
}
// pageCursor is the opaque cursor of a page starting at the i-th step the filters hold (§779): the caller hands it back.
func pageCursor(i int) string {
return base64.RawURLEncoding.EncodeToString([]byte("step:" + strconv.Itoa(i)))
}
// pageFrom is the step a cursor starts at, 0 for none; a cursor this reader did not give is refused.
func pageFrom(next string) (int, error) {
if next = strings.TrimSpace(next); next == "" {
return 0, nil
}
raw, err := base64.RawURLEncoding.DecodeString(next)
if err == nil {
if n, ok := strings.CutPrefix(string(raw), "step:"); ok {
if i, err := strconv.Atoi(n); err == nil && i >= 0 {
return i, nil
}
}
}
return 0, fmt.Errorf("the cursor %q is not one show answered", next)
}
// sizeOf is v's size as JSON.
func sizeOf(v any) int {
b, _ := json.Marshal(v)
return len(b)
}
// Movers is one word per step of each mover of a pipeline (design 47 H2 §789), in the record's movers' order: passed,
// failed, running, waiting or not-run.
func (r Reads) Movers(pipeline string) (MoversAnswer, error) {
p, ok, err := r.pipeline(pipeline)
if err != nil {
return MoversAnswer{}, err
}
if !ok {
return MoversAnswer{}, fmt.Errorf("no pipeline %q in %s", pipeline, seat.StatePipelines)
}
a := MoversAnswer{Pipeline: p.Pipeline, Outcome: p.Outcome, Movers: make([]MoverWords, 0, len(p.Movers))}
at := map[string]int{}
for _, m := range p.Movers {
at[m.Module] = len(a.Movers)
a.Movers = append(a.Movers, MoverWords{Module: m.Module, Steps: map[string]string{}})
}
for _, s := range p.Steps {
i, ok := at[s.Module]
if !ok {
continue
}
name := s.Step
if s.Artifact != "" && s.Step != seat.StepResultStepValidate && s.Step != seat.StepResultStepComposition {
name += " " + s.Artifact
}
a.Movers[i].Steps[name] = StepWord(s.Outcome)
}
return a, nil
}
// MoversAnswer is runner_movers' answer: a pipeline's movers, each with one word per step (design 47 H2 §789).
type MoversAnswer struct {
Pipeline string `json:"pipeline"`
Outcome string `json:"outcome"`
Movers []MoverWords `json:"movers"`
}
// MoverWords is one mover's steps, each by its kind (and artifact where it has one) with one word.
type MoverWords struct {
Module string `json:"module"`
Steps map[string]string `json:"steps"`
}
// StepWord is a step outcome's one word (design 47 H2 §789): passed for one that passed, was carried, accepted, bypassed
// or had nothing to run; failed for one that failed, could not run or did not build; running; waiting for one waiting, on
// a build node too; not-run for one not asked, declined, superseded or cancelled.
func StepWord(outcome string) string {
switch outcome {
case seat.StepResultOutcomePassed, seat.StepResultOutcomeCarried, seat.StepResultOutcomeNothingToRun,
seat.StepResultOutcomeAccepted, seat.StepResultOutcomeBypassed:
return "passed"
case seat.StepResultOutcomeFailed, seat.StepResultOutcomeCouldNotRun, seat.StepResultOutcomeDidNotBuild:
return "failed"
case seat.StepResultOutcomeRunning:
return "running"
case seat.StepResultOutcomeWaiting, seat.StepResultOutcomeWaitingForABuildNode:
return "waiting"
}
return "not-run"
}
// List is the pipelines and runs of a module, a branch, a commit or a run, newest started first, at most Limit when it
// names one (§83): each one's record with its movers and steps counted, its steps empty (H-A-191).
func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
@@ -206,10 +320,6 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error)
if raw == nil {
return seat.Pipeline{}, false, nil
}
raw, err = withoutKept(raw)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err)
}
p, _, err := seat.DecodePipelinesByBranchRevisionValue(raw)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err)
@@ -247,3 +247,70 @@ func TestStandsAndStepStartedNameWhatAReadingStepWaitsOn(t *testing.T) {
t.Error("resume said no step-started")
}
}
// TestAPipelineIsReadInParts holds show and runner_movers to design 47 H2 §789: one mover's steps of a record of 129 movers
// answer under 64 KB; the filters apply while reading; a capped answer continues from its cursor; runner_movers of 1,000
// movers answers under 768 KB with one word per step.
func TestAPipelineIsReadInParts(t *testing.T) {
t0 := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC)
record := func(n int) memKept {
pipes := memKept{}
p := seat.Pipeline{Pipeline: "main.9", Branch: "main", Source: 9, Outcome: seat.PipelineOutcomeRunning, Started: &t0,
Steps: []seat.StepResult{}}
for i := 0; i < n; i++ {
m := fmt.Sprintf("module-%04d", i)
p.Movers = append(p.Movers, seat.Mover{Module: m, Commit: "c9", Fingerprint: "src2:" + strings.Repeat("a", 64)})
var steps []seat.StepResult
for _, k := range []string{seat.StepResultStepBuild, seat.StepResultStepLint, seat.StepResultStepUnit,
seat.StepResultStepIntegration, seat.StepResultStepValidate, seat.StepResultStepComposition} {
o := seat.StepResultOutcomePassed
if k == seat.StepResultStepUnit && i%10 == 3 {
o = seat.StepResultOutcomeFailed
}
steps = append(steps, seat.StepResult{Module: m, Step: k, Artifact: m + ":program", Status: "required", Outcome: o,
Ask: "ask-" + m + "-" + k, Started: &t0, Ended: &t0, Why: strings.Repeat("w", 200)})
}
put(t, pipes, "steps.main.9."+m, seat.PipelinesByStepsPipelineMoverValue{Steps: steps})
}
put(t, pipes, "main.9", p)
return pipes
}
r := Reads{Pipelines: record(129), Kept: memKept{}}
a, err := r.Show(seat.ShowInput{Pipeline: "main.9", Mover: "module-0042"})
if raw, _ := json.Marshal(a); err != nil || len(raw) >= 64<<10 || len(a.Pipeline.Steps) != 6 || a.Total != 6 || a.Capped {
t.Fatalf("one mover of 129: %d bytes, %d steps, total %d, capped %v, %v", len(raw), len(a.Pipeline.Steps), a.Total, a.Capped, err)
}
a, err = r.Show(seat.ShowInput{Pipeline: "main.9", Step: seat.StepResultStepUnit, State: seat.StepResultOutcomeFailed})
if err != nil || a.Total != 13 || len(a.Pipeline.Steps) != 13 || a.Pipeline.Steps[0].Module != "module-0003" {
t.Fatalf("the failed unit steps: total %d, %d steps, %v", a.Total, len(a.Pipeline.Steps), err)
}
// The whole record is larger than one answer: it is read in pages, each under 768 KB, every step once.
big := Reads{Pipelines: record(1000), Kept: memKept{}}
seen, pages, next := 0, 0, ""
for {
a, err := big.Show(seat.ShowInput{Pipeline: "main.9", Next: next})
raw, _ := json.Marshal(a)
if err != nil || len(raw) > MaxAnswer || a.Total != 6000 {
t.Fatalf("page %d: %d bytes, total %d, %v", pages, len(raw), a.Total, err)
}
seen, pages = seen+len(a.Pipeline.Steps), pages+1
if !a.Capped {
break
}
next = a.Next
}
if seen != 6000 || pages < 2 {
t.Fatalf("pages %d read %d steps, want every one of 6000 over more than one page", pages, seen)
}
if _, err := big.Show(seat.ShowInput{Pipeline: "main.9", Next: "not-a-cursor"}); err == nil {
t.Fatal("a cursor show never answered is refused")
}
m, err := big.Movers("main.9")
raw, _ := json.Marshal(m)
if err != nil || len(raw) >= MaxAnswer || len(m.Movers) != 1000 {
t.Fatalf("runner_movers of 1,000: %d bytes, %d movers, %v", len(raw), len(m.Movers), err)
}
if got := m.Movers[3].Steps["unit module-0003:program"] + " " + m.Movers[4].Steps["validate"]; got != "failed passed" {
t.Fatalf("words %q", got)
}
}
@@ -20,6 +20,13 @@ func registrationMissingID(module string) string { return keys.EscapeName(module
// registrationMissing raises registration-missing of module, waited on by the pipeline key (H-A-262).
func (s *Starter) registrationMissing(module, key string) {
// Judged only once the registrations view's first read is in (design 47 H2 §788); before it, said once.
if s.RegistrationsRead != nil && !s.RegistrationsRead() {
if s.unreadSaid.CompareAndSwap(false, true) {
s.logf("registrations not yet read: no step is placed from a registration not read, and no %s is judged before they are (%s waits, in %s)", RegistrationMissingKind, module, key)
}
return
}
if s.Findings == nil {
return
}
@@ -25,7 +25,7 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int,
// A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key
// form a source's pipelines were kept under.
var pkeys []string
for _, f := range []string{b + ".*", b + ".*.published.*.*"} {
for _, f := range []string{b + ".*", b + ".*.published.*.*", b + ".*." + unfinishedToken + ".*"} {
k, err := h.Store.Keys(f)
if err != nil {
return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err)
@@ -1,6 +1,8 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
@@ -189,3 +191,74 @@ func TestAKeptMoverIsPlacedLateAndResumedAtItsKeptRevision(t *testing.T) {
t.Fatalf("Resume asked %d (%v), ids %v: want the kept build %s again", n, err, asks.ids, build.Ask)
}
}
// design 47 H2 §787: a kept mover whose kept revision the runner's view does not hold yet carries unplaced
// kept-revision-unread with that revision, said once in the journal and read with show; once the revision is read it is
// placed and the mark goes.
func TestAKeptMoverWaitingOnItsRevisionSaysSo(t *testing.T) {
reg, entries, _, read := repinFixture()
unread := true
revision := func(branch string, rev int64) (Half, error) {
if unread && rev == 11 {
return Half{}, errors.New("revision 11 is not held yet")
}
return read(branch, rev)
}
store := &MoverStore{Raw: newMemRaw()}
var lines []string
registered := false
s := &Starter{Pipelines: store, Store: store, Steps: &stepAsks{}, Name: "the-runner", Now: ticking(), Revision: revision,
Say: func(string, any) error { return nil }, Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) },
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
key := Key("main", 10)
if ok, err := s.Start(Begin{Branch: "main", Source: 41, Revision: 10, Entries: entries,
Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
if n, err := s.Revised(revisionEvent(11), read); n != 0 || err != nil {
t.Fatalf("the re-pin started %d (%v)", n, err)
}
registered = true
lagging := func(string) (Half, bool) { return Half{Revision: 10, Source: 41, Entries: entries}, true }
for range 2 {
if n, err := s.Registered(store, reg, lagging); n != 0 || err != nil {
t.Fatalf("placed in %d pipelines (%v) while the kept revision is unread", n, err)
}
}
said := 0
for _, l := range lines {
if strings.Contains(l, "app waits for its kept revision 11 to be read") {
said++
}
}
shown, err := Reads{Pipelines: rawKept{store.Raw.(*memRaw)}, Kept: memKept{}}.Show(seat.ShowInput{Pipeline: key})
if err != nil || said != 1 || shown.Pipeline.Movers[0].Unplaced != seat.MoverUnplacedKeptRevisionUnread || shown.Pipeline.Movers[0].Revision != 11 {
t.Fatalf("said %d times, show %+v (%v)", said, shown.Pipeline.Movers, err)
}
if why := s.Unplaced(shown.Pipeline, "app", lagging); why != "app waits for its kept revision 11 to be read" {
t.Fatalf("the unplaced line %q", why)
}
unread = false
if n, err := s.Registered(store, reg, lagging); n != 1 || err != nil {
t.Fatalf("once read: placed in %d pipelines (%v)", n, err)
}
if p, _, _ := store.Get(key); p.Movers[0].Unplaced != "" || p.Movers[0].Revision != 0 {
t.Fatalf("the mark stays once placed: %+v", p.Movers[0])
}
}
// rawKept reads a memRaw as the reads do.
type rawKept struct{ r *memRaw }
func (k rawKept) Keys() ([]string, error) {
var out []string
for key := range k.r.vals {
out = append(out, key)
}
return out, nil
}
func (k rawKept) Value(key string) (json.RawMessage, error) {
v, _, err := k.r.Get(key)
return v, err
}
@@ -9,6 +9,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
@@ -67,9 +68,9 @@ func keyRevision(b Begin) int64 {
return b.Source
}
// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, and a publish's kept from before
// a release revision started one pipeline (H-A-213 (2)).
var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*"}
// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, a publish's kept from before
// a release revision started one pipeline (H-A-213 (2)), and an unfinished mover's pipeline of its own (H-A-398).
var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*", "*.*." + unfinishedToken + ".*"}
// AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83).
func AllKeys(store interface {
@@ -162,6 +163,10 @@ type Starter struct {
// Registration is a module's registration, whose policy names the steps of its pipeline (§21, §23); with none, a
// pipeline starts with no step and says so.
Registration func(module string) (catalog.Registration, bool)
// RegistrationsRead says whether the registrations view's first read is in (design 47 H2 §788): before it no
// registration-missing is judged and "registrations not yet read" is said once; nil reads as read.
RegistrationsRead func() bool
unreadSaid atomic.Bool
// Steps asks each build seat for its step, as the pipeline's asking half (§22, §24); Name is the runner's own name as an
// asker.
Steps StepAsker
@@ -220,6 +225,9 @@ type Starter struct {
// (1), H-A-328): read when a mover's wait on it is written, so a publish made before the wait places the mover without
// any event; nil reads nothing, the publish event alone placing it.
AskPublished func(in builds.PublishedInput) (builds.PublishedAnswer, error)
// Unfinished puts the movers an earlier pipeline left unfinished in the pipeline of the branch's next catalog revision
// (design 47 H2 §786); nil puts none.
Unfinished *Unfinished
publishedMu sync.Mutex
publishedRead map[string]builds.PublishedAnswer
@@ -424,7 +432,7 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
s.logf("the kept-revisions of %s are not read, so its movers are placed as movers not kept: %v", p.Pipeline, err)
}
all := entries
for _, m := range p.Movers {
for i, m := range p.Movers {
// A mover that left for a newer pipeline is placed there, never here (H-A-223, H-A-361).
if placed(*p, m.Module) || slices.Contains(p.Superseded, m.Module) {
continue
@@ -437,10 +445,17 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
// A mover a same-source re-pin kept is placed from the revision it is kept at, its ids its own (H-A-286).
entries := all
if rev := kept[m.Module]; rev > 0 {
if entries, ok = s.keptEntries(p.Branch, rev); !ok {
var err error
if entries, err = s.keptEntries(p.Branch, rev); err != nil {
// It waits for its kept revision to be read, said once and carried on the mover (design 47 H2 §787).
if m.Unplaced != seat.MoverUnplacedKeptRevisionUnread || m.Revision != rev {
s.logf("%s: %s waits for its kept revision %d to be read (%v)", p.Pipeline, m.Module, rev, err)
p.Movers[i].Unplaced, p.Movers[i].Revision = seat.MoverUnplacedKeptRevisionUnread, rev
}
continue
}
}
p.Movers[i].Unplaced, p.Movers[i].Revision = "", 0
// A mover's steps wait for the publish of each mover it builds on (§746, H-A-83 (2)): placed once each is heard,
// ended not asked when one's build ended without passing.
due, waiting, failed := ready(*p, m.Module, entries)
@@ -725,7 +740,9 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
// A kept mover's steps are found by the ids its kept revision gives, from that revision's entries (H-A-286).
entries, rev := half.Entries, kept[m.Module]
if rev > 0 {
if entries, ok = s.keptEntries(p.Branch, rev); !ok {
var err error
if entries, err = s.keptEntries(p.Branch, rev); err != nil {
s.logf("%s: revision %d of %s, which %s is kept at, is not read (%v): its steps are asked again once it is", key, rev, p.Branch, m.Module, err)
continue
}
}
@@ -758,6 +775,10 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
// do not name it (A-327), else "<module> waits on the publish of <package>, …", the packages it awaits not yet published
// as the catalog half of p's branch holds its edges (§746).
func (s *Starter) Unplaced(p seat.Pipeline, module string, view func(branch string) (Half, bool)) string {
// A kept mover waiting for its kept revision says so (design 47 H2 §787).
if m, ok := MoverOf(p, module); ok && m.Unplaced == seat.MoverUnplacedKeptRevisionUnread {
return fmt.Sprintf("%s waits for its kept revision %d to be read", module, m.Revision)
}
if s.Registration == nil {
return module + " has no registration"
}
@@ -880,7 +901,7 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
s.logf("the catalog half of %s holds revision %d: %s waits for revision %d, which its take-in made, to place its steps (§200)", p.Branch, half.Revision, key, p.Revision)
return p, nil, nil, nil, 0, nil
}
before := len(p.Steps)
before, marks := len(p.Steps), unplacedMarks(p)
asks, unregistered, ended, notAsked := s.placeMovers(&p, reg, half.Entries, now)
// Until its registration arrives, the pipeline says it waits on it (A-327): a mover with no registration has no step,
// and it holds the pipeline's end only while every placed mover's required steps passed (H-A-163, H-A-171).
@@ -889,6 +910,15 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
s.registrationMissing(m, key)
}
if len(p.Steps) == before {
// A mover that now waits for its kept revision, or no longer does, is written so (design 47 H2 §787).
if unplacedMarks(p) == marks {
return p, nil, nil, nil, 0, nil
}
if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return p, nil, nil, nil, 0, fmt.Errorf("the movers of %s are not written: %w", key, err)
}
return p, nil, nil, nil, 0, nil
}
if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) {
@@ -901,6 +931,17 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
}
}
// unplacedMarks is what p's movers carry of why they have no step placed (design 47 H2 §787), one line.
func unplacedMarks(p seat.Pipeline) string {
var b strings.Builder
for _, m := range p.Movers {
if m.Unplaced != "" {
fmt.Fprintf(&b, "%s %s %d;", m.Module, m.Unplaced, m.Revision)
}
}
return b.String()
}
// recovered says whether the commit of pipeline key is named a recovery; a read that fails asks on the stream check, said, since
// a step is asked either way (a recovery's only goes ahead of the first round).
func (s *Starter) recovered(key, commit string) bool {
@@ -992,11 +1033,11 @@ func DecodeTakenIn(raw []byte) (catalog.TakeInTaken, error) {
// skip, starts nothing; a second hearing writes nothing. view is the catalog half of the event's branch, read for the
// entries the steps and the tiers are made from.
func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half, bool)) (int, error) {
if len(ev.Entries) == 0 {
s.logf("the take-in of %s source %d names no entries: it starts no pipeline", ev.Branch, ev.Source)
return 0, nil
}
if ev.Branch == "" || ev.Source <= 0 {
if len(ev.Entries) == 0 {
s.logf("the take-in of %s source %d names no entries: it starts no pipeline", ev.Branch, ev.Source)
return 0, nil
}
return 0, fmt.Errorf("a take-in.taken names its watched branch and source revision: branch %q source %d", ev.Branch, ev.Source)
}
// A take-in heard before its branch's half is held starts its pipeline with no steps, never placed from empty entries
@@ -1011,6 +1052,15 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
for _, e := range ev.Entries {
movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository})
}
// The revision the take-in kept carries the movers an earlier pipeline left unfinished (design 47 H2 §786), judged
// against that revision's own entries.
if !ev.Skipped && ev.Revision > 0 {
movers = s.carryUnfinished(ev.Branch, ev.Revision, half, held, movers)
}
if len(movers) == 0 {
s.logf("the take-in of %s source %d names no entries: it starts no pipeline", ev.Branch, ev.Source)
return 0, nil
}
movers = Tiers(movers, half.Entries, s.logf)
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: ev.Call, Revision: ev.Revision, Movers: movers,
Entries: half.Entries, unheld: !held})
@@ -1030,6 +1080,27 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
return started, err
}
// carryUnfinished adds to the movers of revision of branch the unfinished movers whose fingerprint is still their entry's
// there (§786): half when it is that revision, else the revision read by its number; an unread revision carries none, said.
func (s *Starter) carryUnfinished(branch string, revision int64, half Half, held bool, movers []seat.Mover) []seat.Mover {
if s.Unfinished == nil {
return movers
}
if !held || half.Revision != revision {
if s.Revision == nil {
s.logf("revision %d of %s is not read, so it carries no unfinished mover: the runner holds no revision reader", revision, branch)
return movers
}
read, err := s.Revision(branch, revision)
if err != nil {
s.logf("revision %d of %s is not read, so it carries no unfinished mover: %v", revision, branch, err)
return movers
}
half = read
}
return s.Unfinished.Carry(branch, revision, half.Entries, movers)
}
// Release places the movers of pipeline key whose awaited publishes are all heard now (design 47 H §746, H-A-84). It says
// how many pipelines it placed in.
func (s *Starter) Release(store Updater, key string, view func(branch string) (Half, bool)) (int, error) {
@@ -1147,18 +1218,21 @@ func (s *Starter) Revised(ev catalog.CatalogRevisionEvent, read func(branch stri
if half.Revision != ev.Revision {
return 0, fmt.Errorf("the revision verb answered revision %d of %s for %d", half.Revision, branch, ev.Revision)
}
if len(half.Movers) == 0 {
s.logf("the revision %d of %s (%s) names no mover: it starts no pipeline", ev.Revision, branch, ev.Call)
return 0, nil
}
movers := make([]seat.Mover, 0, len(half.Movers))
for _, e := range half.Movers {
movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository})
}
// A revision of the same source moves no mover out of the open pipeline of that source holding it: that pipeline keeps
// it, and this revision starts no pipeline of its own for it (H-A-283); only a revision of a new source moves movers.
if movers = s.keepSameSource(branch, half, movers); len(movers) == 0 {
s.logf("the revision %d of %s (%s) moves only movers the open pipelines of its source %d keep: it starts no pipeline (H-A-283)", ev.Revision, branch, ev.Call, half.Source)
if len(movers) > 0 {
if movers = s.keepSameSource(branch, half, movers); len(movers) == 0 {
s.logf("the revision %d of %s (%s) moves only movers the open pipelines of its source %d keep (H-A-283)", ev.Revision, branch, ev.Call, half.Source)
}
}
// It carries the movers an earlier pipeline left unfinished, or starts a pipeline for them alone (design 47 H2 §786).
movers = s.carryUnfinished(branch, half.Revision, half, true, movers)
if len(movers) == 0 {
s.logf("the revision %d of %s (%s) moves no mover: it starts no pipeline", ev.Revision, branch, ev.Call)
return 0, nil
}
movers = Tiers(movers, half.Entries, s.logf)
@@ -197,6 +197,45 @@ func TestATakeInIsOnePipeline(t *testing.T) {
}
}
// design 47 H2 §788: before the registrations view's first read is in, a start raises no registration-missing, says
// "registrations not yet read" once and places no step; once read, its movers are placed from the registrations held.
func TestTheRegistrationCheckWaitsForItsView(t *testing.T) {
reg, entries := placeFixture()
store := memCreated{memStore{}}
asks := &stepAsks{}
read := false
var lines []string
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(string, any) error { return nil },
Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) },
RegistrationsRead: func() bool { return read },
Registration: func(m string) (catalog.Registration, bool) { return reg, read && m == "app" }}
findings := newHeld()
s.Findings, s.Observer = findings, "the-runner"
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
for _, src := range []int64{4, 5} {
taken := catalog.TakeInTaken{Branch: "main", Source: src, Entries: []catalog.TakeInEntry{{Module: "app", Commit: fmt.Sprint("c", src), Fingerprint: fmt.Sprint("src2:a", src)}}}
if _, err := s.TakenIn(taken, view); err != nil {
t.Fatal(err)
}
}
said := 0
for _, l := range lines {
if strings.HasPrefix(l, "registrations not yet read") {
said++
}
}
if len(findings.held) != 0 || said != 1 || len(asks.asks) != 0 {
t.Fatalf("unread: findings %v, said %d times, %d asks; want none, once, none", findings.held, said, len(asks.asks))
}
read = true
if n, err := s.Waiting(store.memStore, "", view); n == 0 || err != nil || len(asks.asks) == 0 {
t.Fatalf("once read: placed in %d (%v), %d asks", n, err, len(asks.asks))
}
if len(findings.held) != 0 {
t.Fatalf("once read, a registered mover raised %v", findings.held)
}
}
// A-327 (design 47 G §21, §23, §54): a pipeline started before its module has a registration starts with no step; the
// registration-changed that brings it places that pipeline's steps from it and asks them, once: heard again it places nothing.
func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T) {
@@ -392,6 +431,50 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) {
if p, _, err := mem.Get("main.12"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") {
t.Errorf("main.12 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err)
}
// design 47 H2 §786: the next kept revision carries each unfinished mover still at its entry's fingerprint, one journal
// line each; a key the revision moves or holds at another fingerprint, or not at all, goes; a revision that moves
// nothing starts a pipeline for the unfinished alone; place-unfinished puts one in a pipeline of its own, a hand act.
var journal []string
s.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
u := memUnfinished{"main.batched": `{"fingerprint":"src2:bb","revision":11}`, "main.moved": `{"fingerprint":"src2:00"}`,
"main.gone": `{"fingerprint":"src2:cc"}`, "dev.batched": `{"fingerprint":"src2:bb"}`}
s.Unfinished = &Unfinished{Store: u, Logf: s.Logf}
half.Revision = 13
ev.Revision, ev.Source = 13, 10
if n, err := s.TakenIn(ev, view); err != nil || n != 1 {
t.Fatalf("the take-in of revision 13 started %d (%v)", n, err)
}
if p, _, _ := mem.Get("main.13"); len(p.Movers) != 2 || !Moves(p, "moved") || !Moves(p, "batched") {
t.Errorf("main.13 moves %+v, want moved and the unfinished batched", p.Movers)
}
if u.String() != `dev.batched={"fingerprint":"src2:bb"} main.batched={"fingerprint":"src2:bb","revision":11}` {
t.Errorf("the state unfinished = %s, want main.moved (moved at src2:aa) and main.gone (no entry) gone", u.String())
}
if !strings.Contains(strings.Join(journal, "\n"), "revision 13 of main puts batched in a pipeline again") {
t.Errorf("the journal says %q, want batched put in a pipeline again", journal)
}
half.Revision = 14
if n, err := s.TakenIn(catalog.TakeInTaken{Branch: "main", Source: 11, Revision: 14}, view); err != nil || n != 1 {
t.Fatalf("a take-in moving nothing started %d (%v), want one pipeline for the unfinished", n, err)
}
if p, _, _ := mem.Get("main.14"); len(p.Movers) != 1 || !Moves(p, "batched") {
t.Errorf("main.14 moves %+v, want batched alone", p.Movers)
}
half.Revision = 11
acts := &actLog{}
v := Verbs{Starter: s, View: view, Acts: acts}
if a, err := v.PlaceUnfinished("agent.dev", seat.PlaceUnfinishedInput{Key: "main.batched", Why: "now"}); err != nil || a.Pipeline != "main.11.unfinished.batched" || len(*acts) != 1 {
t.Errorf("place-unfinished = %+v (%v), %d hand acts", a, err, len(*acts))
}
if p, _, _ := mem.Get("main.11.unfinished.batched"); len(p.Movers) != 1 || !Moves(p, "batched") {
t.Errorf("main.11.unfinished.batched moves %+v, want batched", p.Movers)
}
for _, in := range []seat.PlaceUnfinishedInput{{Key: "main.batched", Why: "again"}, {Key: "main.none", Why: "x"}, {Key: "dev.batched"}} {
if _, err := v.PlaceUnfinished("agent.dev", in); err == nil {
t.Errorf("place-unfinished %+v is refused", in)
}
}
}
// R16/AC1 (design 47 H §765, H-A-58, H-A-98): a take-in.taken of the old form, with no call (or no branch or source), is
@@ -186,6 +186,9 @@ type Hearer struct {
// Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin);
// nil keys none.
Rejoin func(key string)
// Unfinished keeps, at every end, the movers it leaves unfinished in the state unfinished (design 47 H2 §786); nil keeps
// none.
Unfinished *Unfinished
}
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
@@ -488,6 +491,7 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
h.logf("%s ended %s", p.Pipeline, p.Outcome)
}
h.Failures.Ended(p)
h.Unfinished.Ended(p)
if h.RegistrationWaits != nil {
h.RegistrationWaits()
}
@@ -666,7 +670,7 @@ func stepsOf(p seat.Pipeline, module string) []seat.StepResult {
// pipelineEnded is p's pipeline-ended (design 47 H §746): its key, branch, source, call, revision and outcome, and per
// mover, in the delivery order of its tiers, its module, fingerprint, commit, widens (H-A-119), released version, step results
// and tier.
// and tier; and the movers its end left unfinished (design 47 H2 §786).
func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
movers := make([]seat.MoverEnded, 0, len(p.Movers))
for _, m := range p.Movers {
@@ -676,7 +680,7 @@ func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
sort.SliceStable(movers, func(i, j int) bool { return movers[i].Tier < movers[j].Tier })
return seat.PipelineEnded{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call, Revision: p.Revision,
Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started, Ended: p.Ended, EndCount: p.EndCount, Movers: movers,
Phases: phasesOf(p)}
Phases: phasesOf(p), Unfinished: unfinishedOf(p)}
}
// phasesOf is p's phases (design 47 H2 §785): taken, when the runner wrote the pipeline at its take-in; placed, the
@@ -0,0 +1,222 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"strings"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// StateUnfinished is the runner's state of the movers left unfinished by another mover's failure (design 47 H2 §786, §820):
// one key `<branch>.<module>` per such mover, deleted by the runner or by runner_unfinished_drop.
const StateUnfinished = "unfinished"
// unfinishedToken is the third part of the key of a pipeline of its own of an unfinished mover,
// `<branch>.<revision>.unfinished.<module>` (H-A-398).
const unfinishedToken = "unfinished"
// UnfinishedEntry is one value of the state unfinished (§786): the mover's fingerprint, the catalog revision and the pipeline
// it was left unfinished in, and that pipeline's outcome.
type UnfinishedEntry struct {
Fingerprint string `json:"fingerprint"`
Revision int64 `json:"revision"`
Pipeline string `json:"pipeline"`
Outcome string `json:"outcome"`
}
// UnfinishedStore is the state unfinished: a key's value (nil when it has none), a put, a delete and the keys matching a
// filter, `*` one key part.
type UnfinishedStore interface {
Get(key string) ([]byte, error)
Put(key string, value []byte) error
Delete(key string) error
Keys(filter string) ([]string, error)
}
// UnfinishedKey is a mover's key in the state unfinished, `<branch>.<module>`, each escaped as a key's name is (§786).
func UnfinishedKey(branch, module string) string {
return keys.EscapeName(branch) + "." + keys.EscapeName(module)
}
// UnfinishedPipelineKey is the key of an unfinished mover's pipeline of its own, `<branch>.<revision>.unfinished.<module>`
// (H-A-398).
func UnfinishedPipelineKey(branch string, revision int64, module string) string {
return seat.PipelinesByBranchRevisionUnfinishedModuleKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10), keys.EscapeName(module))
}
// neverRan are the outcomes of a step that never ran: still waiting, not asked, cancelled, held by a dependency that did not
// build, or waiting for a build node (§786).
var neverRan = []string{seat.StepResultOutcomeWaiting, seat.StepResultOutcomeNotAsked, seat.StepResultOutcomeCancelled,
seat.StepResultOutcomeDidNotBuild, seat.StepResultOutcomeWaitingForABuildNode}
// leftUnfinished says whether a pipeline's end leaves movers unfinished: a delivering pipeline (no run on request) that ended
// failed or cancelled (§786; a could-not-run step ends a pipeline failed, H-A-398).
func leftUnfinished(p seat.Pipeline) bool {
return p.Run == "" && (p.Outcome == seat.PipelineOutcomeFailed || p.Outcome == seat.PipelineOutcomeCancelled)
}
// unfinishedOf are the movers p's end leaves unfinished (§786): in a pipeline ended failed or cancelled, each mover no newer
// revision moved out of it whose own required steps all passed or never ran.
func unfinishedOf(p seat.Pipeline) []string {
if !leftUnfinished(p) {
return nil
}
var out []string
for _, m := range p.Movers {
if slices.Contains(p.Superseded, m.Module) {
continue
}
if ownStepsHeld(p, m.Module) {
out = append(out, m.Module)
}
}
return out
}
// ownStepsHeld says whether every required step of module in p passed or never ran.
func ownStepsHeld(p seat.Pipeline, module string) bool {
for _, s := range p.Steps {
if s.Module != module || s.Status != seat.StepResultStatusRequired {
continue
}
if !stepPassed(s) && !slices.Contains(neverRan, s.Outcome) {
return false
}
}
return true
}
// Unfinished keeps the movers left unfinished in the state unfinished and puts them in a pipeline again (§786).
type Unfinished struct {
Store UnfinishedStore
Logf func(string, ...any)
}
func (u *Unfinished) logf(format string, args ...any) {
if u.Logf != nil {
u.Logf(format, args...)
}
}
// read is one key's entry, and whether it is held.
func (u *Unfinished) read(key string) (UnfinishedEntry, bool, error) {
raw, err := u.Store.Get(key)
if err != nil || raw == nil {
return UnfinishedEntry{}, false, err
}
var e UnfinishedEntry
if err := json.Unmarshal(raw, &e); err != nil {
return UnfinishedEntry{}, false, fmt.Errorf("the unfinished entry %s is not read: %w", key, err)
}
return e, true, nil
}
// Ended takes a pipeline's end into the state unfinished (§786): a mover p leaves unfinished is kept under its key; a mover
// whose pipeline ended passed, or whose own required step did not pass (it waits for its fingerprint to move), has its key
// deleted when the key names its fingerprint. A run on request's pipeline, and one every mover left, change nothing.
func (u *Unfinished) Ended(p seat.Pipeline) {
if u == nil || p.Run != "" {
return
}
switch p.Outcome {
case seat.PipelineOutcomePassed, seat.PipelineOutcomeFailed, seat.PipelineOutcomeCancelled:
default:
return
}
left := unfinishedOf(p)
for _, m := range p.Movers {
if slices.Contains(p.Superseded, m.Module) {
continue
}
key := UnfinishedKey(p.Branch, m.Module)
if slices.Contains(left, m.Module) {
raw, _ := json.Marshal(UnfinishedEntry{Fingerprint: m.Fingerprint, Revision: p.Revision, Pipeline: p.Pipeline, Outcome: p.Outcome})
if err := u.Store.Put(key, raw); err != nil {
u.logf("%s ended %s and leaves %s unfinished, but it is not kept in the state unfinished: %v", p.Pipeline, p.Outcome, m.Module, err)
continue
}
u.logf("%s ended %s and leaves %s unfinished: it is put in a pipeline again at %s's next catalog revision (§786)", p.Pipeline, p.Outcome, m.Module, p.Branch)
continue
}
e, held, err := u.read(key)
if err != nil {
u.logf("%s: the unfinished entry of %s is not read: %v", p.Pipeline, m.Module, err)
continue
}
if !held || e.Fingerprint != m.Fingerprint {
continue
}
if err := u.Store.Delete(key); err != nil {
u.logf("%s: the unfinished entry of %s is not deleted: %v", p.Pipeline, m.Module, err)
continue
}
u.logf("%s ended %s: %s is no longer unfinished", p.Pipeline, p.Outcome, m.Module)
}
}
// Carry adds to the movers of a catalog revision of branch each unfinished mover whose fingerprint is still its entry's in
// entries, the revision's entries, with one journal line per mover added again (§786). An unfinished mover the revision
// moves at a newer fingerprint, or whose entry holds another fingerprint or none, has its key deleted; one it moves at its
// own fingerprint is carried by the revision already. It answers movers with the added ones after them.
func (u *Unfinished) Carry(branch string, revision int64, entries map[string]catalog.CatalogEntry, movers []seat.Mover) []seat.Mover {
if u == nil {
return movers
}
ukeys, err := u.Store.Keys(keys.EscapeName(branch) + ".*")
if err != nil {
u.logf("the unfinished movers of %s are not listed, so revision %d carries none: %v", branch, revision, err)
return movers
}
slices.Sort(ukeys)
for _, key := range ukeys {
_, name, _ := strings.Cut(key, ".")
module, err := keys.UnescapeName(name)
if err != nil {
continue
}
e, held, err := u.read(key)
if err != nil || !held {
if err != nil {
u.logf("revision %d of %s: %v", revision, branch, err)
}
continue
}
if i := slices.IndexFunc(movers, func(m seat.Mover) bool { return m.Module == module }); i >= 0 {
if movers[i].Fingerprint != e.Fingerprint {
u.dropMoved(key, module, fmt.Sprintf("revision %d of %s moves it at %s", revision, branch, movers[i].Fingerprint))
}
continue
}
entry, ok := entries[module]
if !ok || entry.Fp != e.Fingerprint {
u.dropMoved(key, module, fmt.Sprintf("revision %d of %s holds it at %q", revision, branch, entry.Fp))
continue
}
movers = append(movers, seat.Mover{Module: module, Commit: entry.Commit, Fingerprint: entry.Fp, Repository: entry.Repository})
u.logf("revision %d of %s puts %s in a pipeline again: %s left it unfinished (%s) at fingerprint %s (§786)", revision, branch, module, e.Pipeline, e.Outcome, e.Fingerprint)
}
return movers
}
// dropMoved deletes the key of an unfinished mover a newer fingerprint moves (§786).
func (u *Unfinished) dropMoved(key, module, why string) {
if err := u.Store.Delete(key); err != nil {
u.logf("the unfinished entry of %s is not deleted: %v", module, err)
return
}
u.logf("%s is no longer unfinished: %s", module, why)
}
// Entry is one key's entry; false when the state holds none.
func (u *Unfinished) Entry(key string) (UnfinishedEntry, bool, error) {
if u == nil {
return UnfinishedEntry{}, false, errors.New("the runner keeps no state unfinished")
}
return u.read(key)
}
@@ -594,8 +594,9 @@ func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time
// A kept mover's step is the one placed from the revision it is kept at, under its kept ids (H-A-286).
entries, rev := half.Entries, v.keptOf(p.Pipeline)[m.Module]
if rev > 0 {
if entries, ok = v.Starter.keptEntries(p.Branch, rev); !ok {
return nil, fmt.Errorf("revision %d of %s, which %s is kept at, is not read: the step is asked when it is", rev, p.Branch, m.Module)
var err error
if entries, err = v.Starter.keptEntries(p.Branch, rev); err != nil {
return nil, fmt.Errorf("revision %d of %s, which %s is kept at, is not read (%v): the step is asked when it is", rev, p.Branch, m.Module, err)
}
}
_, placed := place(p, m, reg, entries, now, v.Starter.Name, false, rev)
@@ -726,3 +727,63 @@ func (v Verbs) Place(caller string, in seat.PlaceInput) (seat.PlaceAnswer, error
p, _, err = s.Store.Get(in.Pipeline)
return seat.PlaceAnswer{Pipeline: p}, err
}
// PlaceUnfinished puts a mover kept in the state unfinished in a pipeline of its own at once, recorded in the hand-act log
// first (design 47 H2 §786, H-A-398): keyed `<branch>.<revision>.unfinished.<module>`, its steps placed from the entries of
// the catalog revision the entry names, the mover at the entry's fingerprint. The key stays until that pipeline ends passed.
// A key the state does not hold, a revision whose entry holds another fingerprint, or a pipeline written already is refused,
// nothing done.
func (v Verbs) PlaceUnfinished(_ string, in seat.PlaceUnfinishedInput) (seat.PlaceUnfinishedAnswer, error) {
s := v.Starter
if strings.TrimSpace(in.Key) == "" {
return seat.PlaceUnfinishedAnswer{}, errors.New("place-unfinished names the unfinished entry's key")
}
if s == nil || s.Unfinished == nil {
return seat.PlaceUnfinishedAnswer{}, errors.New("the runner keeps no state unfinished: nothing was placed")
}
e, held, err := s.Unfinished.Entry(in.Key)
if err != nil {
return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the unfinished entry %s is not read: %w", in.Key, err)
}
if !held {
return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the state unfinished holds no %s: nothing was placed", in.Key)
}
b, m, _ := strings.Cut(in.Key, ".")
branch, berr := keys.UnescapeName(b)
module, merr := keys.UnescapeName(m)
if berr != nil || merr != nil || branch == "" || module == "" {
return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("%s is no key <branch>.<module>: nothing was placed", in.Key)
}
var half Half
switch {
case s.Revision != nil:
if half, err = s.Revision(branch, e.Revision); err != nil {
return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the revision %d of %s is not read: nothing was placed: %w", e.Revision, branch, err)
}
case v.View != nil:
var ok bool
if half, ok = v.View(branch); !ok || half.Revision != e.Revision {
return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the revision %d of %s is not held: nothing was placed", e.Revision, branch)
}
default:
return seat.PlaceUnfinishedAnswer{}, errors.New("the runner reads no catalog revision: nothing was placed")
}
entry, ok := half.Entries[module]
if !ok || entry.Fp != e.Fingerprint || half.Source <= 0 {
return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the revision %d of %s holds %s at %q, not its unfinished fingerprint %s: nothing was placed", e.Revision, branch, module, entry.Fp, e.Fingerprint)
}
key := UnfinishedPipelineKey(branch, e.Revision, module)
if err := v.record(seat.VerbPlaceUnfinished, "put the unfinished mover "+module+" of "+branch+" in the pipeline "+key+" by hand", in.Why); err != nil {
return seat.PlaceUnfinishedAnswer{}, err
}
started, err := s.Start(Begin{Branch: branch, Source: half.Source, Revision: e.Revision, Entries: half.Entries, Key: key,
Movers: []seat.Mover{{Module: module, Commit: entry.Commit, Fingerprint: entry.Fp, Repository: entry.Repository, Tier: 1}}})
if err != nil {
return seat.PlaceUnfinishedAnswer{}, err
}
if !started {
return seat.PlaceUnfinishedAnswer{}, fmt.Errorf("the pipeline %s is started already: nothing was placed", key)
}
s.logf("%s puts the unfinished mover %s in a pipeline of its own by hand (§786)", key, module)
return seat.PlaceUnfinishedAnswer{Pipeline: key}, nil
}
@@ -55,7 +55,9 @@ func TestManifest(t *testing.T) {
{"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat &&
equalSets(m.Claims[0].Serves, seat.Verbs())},
// §27, §85, §330, §355, §77: the old state sections.
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers", seat.StateQueue})},
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers", seat.StateQueue, "unfinished"})},
// design 47 H2 §786, §820: unfinished is a state of 8 MiB.
{"unfinished 8 MiB", stateMiB["unfinished"] == 8},
// §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live.
{"bounds is key-ttl", keyTTL["bounds"]},
// §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling.
@@ -117,7 +119,9 @@ func TestManifest(t *testing.T) {
// RN25 (§539): the runner's tools carry the prefix runner_.
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append",
"runner_start", "runner_run", "runner_accept", "runner_ask_again", "runner_place", "runner_end_again", "runner_waiting",
"runner_test_runs", "runner_test_runs_drop"})},
"runner_test_runs", "runner_test_runs_drop", "runner_unfinished", "runner_unfinished_drop", "runner_unfinished_place",
// design 47 H2 §789: a pipeline read in parts.
"runner_show", "runner_movers"})},
// A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery,
// never by a tool of its own; runner_health stays a tool.
{"health is the unit", healthTool(raw) == "unit"},
@@ -203,7 +207,15 @@ func runSettingsOf(raw []byte) bool {
}
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "ask-silent-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
"pipelines-at-once": "number", "pipelines-waiting-cap": "number"}
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
// design 47 H2 §790, §781: stalled-bound is read through mesh-sdk's settings reader, by the process and the tools
// program alike, from the words the node-engine composes: no file of its own.
words := map[string]string{"stalled-bound": "string"}
for key, kind := range words {
if m.Config[key]["type"] != kind || m.Config[key]["default"] != "5m" || m.Config[key]["runtime"] != true {
return false
}
}
if len(m.Config) != len(want)+len(words) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) ||
m.Config["pipelines-at-once"]["default"] != float64(1) || m.Config["pipelines-waiting-cap"]["default"] != float64(64) {
return false
+19 -1
View File
@@ -20,6 +20,7 @@
"cancel",
"end-again",
"place",
"place-unfinished",
"ask-again",
"test-runs",
"test-runs-drop"
@@ -40,7 +41,12 @@
"runner_end_again",
"runner_waiting",
"runner_test_runs",
"runner_test_runs_drop"
"runner_test_runs_drop",
"runner_unfinished",
"runner_unfinished_drop",
"runner_unfinished_place",
"runner_show",
"runner_movers"
],
"consumes": [
"mesh-delivery-indexer.take-in.taken",
@@ -161,6 +167,13 @@
"trusted": false,
"default": "10m"
},
"stalled-bound": {
"type": "string",
"description": "how long a running pipeline may have nothing open and nothing newly placed, or a step's ask stay untaken, before the runner's health names it, a Go duration such as 5m (design 47 H2 sections 778, 790).",
"trusted": false,
"runtime": true,
"default": "5m"
},
"ask-silent-bound": {
"type": "string",
"description": "how long an ask the runner puts on a seat whose holder says nothing before the step's end (mesh-builds, mesh-wants) may go without that end before its step ends could not run, a Go duration such as 45m (design 47 H H-A-243).",
@@ -217,6 +230,11 @@
{
"name": "consumers",
"history": 1
},
{
"name": "unfinished",
"history": 1,
"max-mib": 8
}
],
"streams": [