From 9ab1c3b8f38e309e7aad73061e93540dea32732c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:08:40 +0100 Subject: [PATCH] H-A-375: a declined pipeline step is put again on a node that has its need MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A holder whose node lacks what a step's packages need (the artifact's check-needs, now named on the integration step's ask) says the step's end declined with needs and lacks, in place of could not run. The runner puts it again under a new ask id naming in not-on the nodes that declined it; a holder on a listed node hands it back at once. The nodes that can take it are the holders of the step's seat and of the need's seat less those that declined; when none is left the step ends could not run naming the need and the nodes asked. The build-step part's form is 3, the runner seat's 5. Builds: design 47 H2 H-A-375, H-A-445 (ADR 0552 §2, §4) --- .../go/seats/meshpipelinerunner/contract.go | 8 +- .../go/seats/nodebuildgo/contract.go | 24 ++-- .../go/seats/nodebuildimage/contract.go | 24 ++-- .../go/seats/nodebuildnode/contract.go | 24 ++-- .../parts/build-step.json | 31 +++- .../seats/mesh-pipeline-runner/seat.json | 9 +- .../ts/seats/meshpipelinerunner/index.d.ts | 2 + .../seats/nodebuildgo/parts/build-step.d.ts | 11 +- .../nodebuildimage/parts/build-step.d.ts | 11 +- .../seats/nodebuildnode/parts/build-step.d.ts | 11 +- .../cmd/mesh-delivery-pipeline/main.go | 2 + .../cmd/mesh-delivery-pipeline/views.go | 17 +++ .../internal/pipeline/declined.go | 136 ++++++++++++++++++ .../internal/pipeline/place.go | 32 ++++- .../internal/pipeline/place_test.go | 23 +++ .../internal/pipeline/start.go | 10 ++ .../internal/pipeline/steps.go | 27 +++- .../internal/pipeline/steps_test.go | 38 +++++ .../internal/pipeline/verbs.go | 15 ++ modules/mesh-delivery-sdk/buildstep/ask.go | 11 +- .../buildstep/pipelinestep.go | 50 ++++--- modules/mesh-delivery-sdk/steprun/askers.go | 20 +++ .../mesh-delivery-sdk/steprun/askrun_test.go | 13 ++ modules/mesh-delivery-sdk/steprun/pull.go | 27 +++- .../steprun/pull_poison_test.go | 48 ++++++- modules/mesh-delivery-sdk/steprun/refusal.go | 3 + modules/mesh-delivery-sdk/steprun/steprun.go | 13 +- 27 files changed, 567 insertions(+), 73 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/declined.go diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index 9dc9ac446..4067876f5 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -14,13 +14,13 @@ const Seat = "mesh-pipeline-runner" // Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780): // a reader of a value of another form says "form N, this reader knows M". -const Form = 4 +const Form = 5 // The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition // whose fields changed while its forms did not (design 47 H2 §780). const ( - generatedForms = "4" - generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=ab5b72044ab6 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=e1947a0a8dca StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466" + generatedForms = "5" + generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=ab5b72044ab6 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466" ) const ( @@ -906,6 +906,8 @@ type StepResult struct { Module string `json:"module,omitempty"` // the node whose holder ended it Node string `json:"node,omitempty"` + // the nodes that declined the step's ask, lacking what it needs: its ask is put again under a new id naming them, each node declining at most once (design 47 H2 H-A-375, H-A-445) + NotOn []string `json:"not-on,omitempty"` // for a waiting step, who or what it waits on: a person, a build node, the revision view On string `json:"on,omitempty"` // never passed for a step that ran nothing (design 47 *Pipeline steps and the policy*) diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go index 1744be5f1..bc85bbf36 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go @@ -17,13 +17,13 @@ const Seat = "node-build-go" const Form = 1 // FormBuildStep is the form of what the part build-step adds. -const FormBuildStep = 2 +const FormBuildStep = 3 // The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition // whose fields changed while its forms did not (design 47 H2 §780). const ( - generatedForms = "1 build-step:2" - generatedFieldsDigest = "AnswerAnswer=a3b42bd5e791 AskArtifact=35eb85c27e97 AskTaken=8c917075eed4 AskTakenEvent=e4fc9b81e066 BuiltArtifact=b5fcef7a8b58 BuiltArtifactResolvedItem=3bcbbcfd1a1f Capacity=b40cca7ccdfa CheckAccept=ef89c6962ae4 CheckRunAccept=ef89c6962ae4 CheckStepAccept=ef89c6962ae4 CurrentAnswer=e39824e012cb CurrentAsk=ac167e5ee11f DeliveryAccept=ef89c6962ae4 DeliveryStepAccept=ef89c6962ae4 DetectedStep=a6fcd7d0eca4 DetectedSteps=05198185c776 DetectedStepsByModuleArtifactValue=cb35b5775cc6 Drop=fb4637978e86 DropAnswer=7538f2f5e68e FailingTest=f3fb3f1a3ca8 Fetch=6e12677bbc2c FetchAnswer=46fc3d5bd7ee FetchBesideItem=774b4bf88ba4 FetchBuildsItem=aee5adc97246 FetchPackage=20d1668a0eee FetchRefusal=de1518848b4c InstallAsk=9f965aa5b30b KeptArtifactAnswer=706729f8ccb9 KeptResult=8657683cdcd7 LevelAnswer=143c69c28a6a LevelAnswerDigest=d54b7fd7b1da LevelAnswerDigestNeedsItem=946b37d1d4a1 LevelAnswerDigestPackagesItem=a7d30558470d LevelAnswerKeptItem=85d3c2f78632 LevelAnswerUnmetItem=946b37d1d4a1 ModuleTree=00bb59e9f262 NotRun=d322fb1eb153 OutputAnswer=97ba4bd92bf9 Pause=d6521e0cdd1e PauseAnswer=7538f2f5e68e PausesByCapacityNodeValue=5392cf505c80 PausesByDroppedIDValue=0e15ac9cacbd PausesByPausedNodeValue=d111aad0f0cf PlacedBeside=53479337ff18 Program=b48ae74f34e0 RecheckAccept=ef89c6962ae4 ResumeAnswer=7538f2f5e68e RunHead=774b4bf88ba4 RunInput=3dc41505a4b7 RunJob=88af2a7f432b RunProgram=6fd636999d54 Step=75190430d510 StepAsk=a22b1f717fc3 StepAskBuild=de784b6c05da StepAskCheck=b77941df6dea StepAskCompose=4c6a08e10231 StepEnded=14b484f0ffa2 StepEndedEvent=207787118dac StepProgress=d25781cc895a StepProgressEvent=179344ce0ca2 StepResult=67f88be36d0e StepRun=a7a81f25515f StepsAnswer=2c02cac2df23 StepsByAskerNodeIDValue=50ccfd4cebd8 StepsByNodeIDValue=50ccfd4cebd8 TakenVersion=1b8d0547f61e TestCounts=9c1332427159 Toolchain=8ecb7debf9ad ToolchainReported=d26392767c50 ToolchainReportedEvent=fa0a431c9c7b ToolchainUsed=6cc7ccabc9f8 ToolchainsByNodeLanguageValue=f97d1dd27613" + generatedForms = "1 build-step:3" + generatedFieldsDigest = "AnswerAnswer=a3b42bd5e791 AskArtifact=35eb85c27e97 AskTaken=8c917075eed4 AskTakenEvent=e4fc9b81e066 BuiltArtifact=b5fcef7a8b58 BuiltArtifactResolvedItem=3bcbbcfd1a1f Capacity=b40cca7ccdfa CheckAccept=ef89c6962ae4 CheckRunAccept=ef89c6962ae4 CheckStepAccept=ef89c6962ae4 CurrentAnswer=e39824e012cb CurrentAsk=ac167e5ee11f DeliveryAccept=ef89c6962ae4 DeliveryStepAccept=ef89c6962ae4 DetectedStep=a6fcd7d0eca4 DetectedSteps=05198185c776 DetectedStepsByModuleArtifactValue=cb35b5775cc6 Drop=fb4637978e86 DropAnswer=7538f2f5e68e FailingTest=f3fb3f1a3ca8 Fetch=6e12677bbc2c FetchAnswer=46fc3d5bd7ee FetchBesideItem=774b4bf88ba4 FetchBuildsItem=aee5adc97246 FetchPackage=20d1668a0eee FetchRefusal=de1518848b4c InstallAsk=9f965aa5b30b KeptArtifactAnswer=706729f8ccb9 KeptResult=8657683cdcd7 LevelAnswer=143c69c28a6a LevelAnswerDigest=d54b7fd7b1da LevelAnswerDigestNeedsItem=946b37d1d4a1 LevelAnswerDigestPackagesItem=a7d30558470d LevelAnswerKeptItem=85d3c2f78632 LevelAnswerUnmetItem=946b37d1d4a1 ModuleTree=00bb59e9f262 NotRun=d322fb1eb153 OutputAnswer=97ba4bd92bf9 Pause=d6521e0cdd1e PauseAnswer=7538f2f5e68e PausesByCapacityNodeValue=5392cf505c80 PausesByDroppedIDValue=0e15ac9cacbd PausesByPausedNodeValue=d111aad0f0cf PlacedBeside=53479337ff18 Program=b48ae74f34e0 RecheckAccept=ef89c6962ae4 ResumeAnswer=7538f2f5e68e RunHead=774b4bf88ba4 RunInput=3dc41505a4b7 RunJob=88af2a7f432b RunProgram=6fd636999d54 Step=75190430d510 StepAsk=f0d09ea1670e StepAskBuild=de784b6c05da StepAskCheck=b77941df6dea StepAskCompose=4c6a08e10231 StepEnded=9bc24593a47d StepEndedEvent=207787118dac StepProgress=d25781cc895a StepProgressEvent=179344ce0ca2 StepResult=67f88be36d0e StepRun=a7a81f25515f StepsAnswer=2c02cac2df23 StepsByAskerNodeIDValue=50ccfd4cebd8 StepsByNodeIDValue=50ccfd4cebd8 TakenVersion=1b8d0547f61e TestCounts=9c1332427159 Toolchain=8ecb7debf9ad ToolchainReported=d26392767c50 ToolchainReportedEvent=fa0a431c9c7b ToolchainUsed=6cc7ccabc9f8 ToolchainsByNodeLanguageValue=f97d1dd27613" ) const ( @@ -808,8 +808,11 @@ type StepAsk struct { // build, check, compose, step or install Kind string `json:"kind"` // go, typescript or image - Language string `json:"language"` - Needs []string `json:"needs,omitempty"` + Language string `json:"language"` + // what the ask needs of the holder's node, by seat name: a check's packages' needs, and a pipeline step's, read from its artifact's check-needs in the module's manifest; a holder whose node lacks one declines the ask (ADR 0552 sections 2, 4; design 47 H2 H-A-375) + Needs []string `json:"needs,omitempty"` + // the nodes that declined the step before, to which the runner puts it again under a new id: a holder on a listed node hands it back at once, unacknowledged, for another node to take (design 47 H2 H-A-445) + NotOn []string `json:"not-on,omitempty"` // package to the tagged version a build pins Packages map[string]string `json:"packages,omitempty"` // a run: a program built from a change and run in the build seat's sandbox over the heads, its input asked of a verb of the asker's seat; the Go seat's check stream alone carries it, as a compose @@ -847,13 +850,17 @@ type StepEnded struct { // the ask's id ID string `json:"id"` // the packages that passed at their digest, for the runner to keep - Kept []KeptResult `json:"kept,omitempty"` - Module string `json:"module,omitempty"` + Kept []KeptResult `json:"kept,omitempty"` + // for a declined step, what the node lacked of them (design 47 H2 H-A-445) + Lacks []string `json:"lacks,omitempty"` + Module string `json:"module,omitempty"` + // for a declined step, the seats the package needs the node to hold (design 47 H2 H-A-445) + Needs []string `json:"needs,omitempty"` // the node that ran it, as in the subject Node string `json:"node"` // the tests that did not run, each with its reason, counted apart from passed and failed (design 47 H2 §800) NotRun []NotRun `json:"not-run,omitempty"` - // never passed for a step that ran nothing + // never passed for a step that ran nothing; declined for a step the holder's node lacks a need of, with needs and lacks, which the runner puts again on a node that has it (design 47 H2 H-A-375, H-A-445) Outcome string `json:"outcome"` // the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) Output string `json:"output,omitempty"` @@ -874,6 +881,7 @@ const ( StepEndedOutcomeFailed = "failed" StepEndedOutcomeNothingToRun = "nothing to run" StepEndedOutcomeCouldNotRun = "could not run" + StepEndedOutcomeDeclined = "declined" ) // The values of StepEnded's step. diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go index 5d2ecb30f..58001b46b 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go @@ -17,13 +17,13 @@ const Seat = "node-build-image" const Form = 1 // FormBuildStep is the form of what the part build-step adds. -const FormBuildStep = 2 +const FormBuildStep = 3 // The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition // whose fields changed while its forms did not (design 47 H2 §780). const ( - generatedForms = "1 build-step:2" - generatedFieldsDigest = "AnswerAnswer=a3b42bd5e791 AskArtifact=35eb85c27e97 AskTaken=8c917075eed4 AskTakenEvent=e4fc9b81e066 BuiltArtifact=b5fcef7a8b58 BuiltArtifactResolvedItem=3bcbbcfd1a1f Capacity=b40cca7ccdfa CheckAccept=ef89c6962ae4 CheckStepAccept=ef89c6962ae4 CurrentAnswer=e39824e012cb CurrentAsk=ac167e5ee11f DeliveryAccept=ef89c6962ae4 DeliveryStepAccept=ef89c6962ae4 DetectedStep=a6fcd7d0eca4 DetectedSteps=05198185c776 DetectedStepsByModuleArtifactValue=cb35b5775cc6 Drop=fb4637978e86 DropAnswer=7538f2f5e68e FailingTest=f3fb3f1a3ca8 Fetch=6e12677bbc2c FetchAnswer=46fc3d5bd7ee FetchBesideItem=774b4bf88ba4 FetchBuildsItem=aee5adc97246 FetchPackage=20d1668a0eee FetchRefusal=de1518848b4c InstallAsk=9f965aa5b30b KeptArtifactAnswer=706729f8ccb9 KeptResult=8657683cdcd7 LevelAnswer=143c69c28a6a LevelAnswerDigest=d54b7fd7b1da LevelAnswerDigestNeedsItem=946b37d1d4a1 LevelAnswerDigestPackagesItem=a7d30558470d LevelAnswerKeptItem=85d3c2f78632 LevelAnswerUnmetItem=946b37d1d4a1 ModuleTree=00bb59e9f262 NotRun=d322fb1eb153 OutputAnswer=97ba4bd92bf9 Pause=d6521e0cdd1e PauseAnswer=7538f2f5e68e PausesByCapacityNodeValue=5392cf505c80 PausesByDroppedIDValue=0e15ac9cacbd PausesByPausedNodeValue=d111aad0f0cf PlacedBeside=53479337ff18 Program=b48ae74f34e0 RecheckAccept=ef89c6962ae4 ResumeAnswer=7538f2f5e68e RunHead=774b4bf88ba4 RunInput=3dc41505a4b7 RunJob=88af2a7f432b RunProgram=6fd636999d54 Step=75190430d510 StepAsk=a22b1f717fc3 StepAskBuild=de784b6c05da StepAskCheck=b77941df6dea StepAskCompose=4c6a08e10231 StepEnded=14b484f0ffa2 StepEndedEvent=207787118dac StepProgress=d25781cc895a StepProgressEvent=179344ce0ca2 StepResult=67f88be36d0e StepRun=a7a81f25515f StepsAnswer=2c02cac2df23 StepsByAskerNodeIDValue=50ccfd4cebd8 StepsByNodeIDValue=50ccfd4cebd8 TakenVersion=1b8d0547f61e TestCounts=9c1332427159 Toolchain=8ecb7debf9ad ToolchainReported=d26392767c50 ToolchainReportedEvent=fa0a431c9c7b ToolchainUsed=6cc7ccabc9f8 ToolchainsByNodeLanguageValue=f97d1dd27613" + generatedForms = "1 build-step:3" + generatedFieldsDigest = "AnswerAnswer=a3b42bd5e791 AskArtifact=35eb85c27e97 AskTaken=8c917075eed4 AskTakenEvent=e4fc9b81e066 BuiltArtifact=b5fcef7a8b58 BuiltArtifactResolvedItem=3bcbbcfd1a1f Capacity=b40cca7ccdfa CheckAccept=ef89c6962ae4 CheckStepAccept=ef89c6962ae4 CurrentAnswer=e39824e012cb CurrentAsk=ac167e5ee11f DeliveryAccept=ef89c6962ae4 DeliveryStepAccept=ef89c6962ae4 DetectedStep=a6fcd7d0eca4 DetectedSteps=05198185c776 DetectedStepsByModuleArtifactValue=cb35b5775cc6 Drop=fb4637978e86 DropAnswer=7538f2f5e68e FailingTest=f3fb3f1a3ca8 Fetch=6e12677bbc2c FetchAnswer=46fc3d5bd7ee FetchBesideItem=774b4bf88ba4 FetchBuildsItem=aee5adc97246 FetchPackage=20d1668a0eee FetchRefusal=de1518848b4c InstallAsk=9f965aa5b30b KeptArtifactAnswer=706729f8ccb9 KeptResult=8657683cdcd7 LevelAnswer=143c69c28a6a LevelAnswerDigest=d54b7fd7b1da LevelAnswerDigestNeedsItem=946b37d1d4a1 LevelAnswerDigestPackagesItem=a7d30558470d LevelAnswerKeptItem=85d3c2f78632 LevelAnswerUnmetItem=946b37d1d4a1 ModuleTree=00bb59e9f262 NotRun=d322fb1eb153 OutputAnswer=97ba4bd92bf9 Pause=d6521e0cdd1e PauseAnswer=7538f2f5e68e PausesByCapacityNodeValue=5392cf505c80 PausesByDroppedIDValue=0e15ac9cacbd PausesByPausedNodeValue=d111aad0f0cf PlacedBeside=53479337ff18 Program=b48ae74f34e0 RecheckAccept=ef89c6962ae4 ResumeAnswer=7538f2f5e68e RunHead=774b4bf88ba4 RunInput=3dc41505a4b7 RunJob=88af2a7f432b RunProgram=6fd636999d54 Step=75190430d510 StepAsk=f0d09ea1670e StepAskBuild=de784b6c05da StepAskCheck=b77941df6dea StepAskCompose=4c6a08e10231 StepEnded=9bc24593a47d StepEndedEvent=207787118dac StepProgress=d25781cc895a StepProgressEvent=179344ce0ca2 StepResult=67f88be36d0e StepRun=a7a81f25515f StepsAnswer=2c02cac2df23 StepsByAskerNodeIDValue=50ccfd4cebd8 StepsByNodeIDValue=50ccfd4cebd8 TakenVersion=1b8d0547f61e TestCounts=9c1332427159 Toolchain=8ecb7debf9ad ToolchainReported=d26392767c50 ToolchainReportedEvent=fa0a431c9c7b ToolchainUsed=6cc7ccabc9f8 ToolchainsByNodeLanguageValue=f97d1dd27613" ) const ( @@ -806,8 +806,11 @@ type StepAsk struct { // build, check, compose, step or install Kind string `json:"kind"` // go, typescript or image - Language string `json:"language"` - Needs []string `json:"needs,omitempty"` + Language string `json:"language"` + // what the ask needs of the holder's node, by seat name: a check's packages' needs, and a pipeline step's, read from its artifact's check-needs in the module's manifest; a holder whose node lacks one declines the ask (ADR 0552 sections 2, 4; design 47 H2 H-A-375) + Needs []string `json:"needs,omitempty"` + // the nodes that declined the step before, to which the runner puts it again under a new id: a holder on a listed node hands it back at once, unacknowledged, for another node to take (design 47 H2 H-A-445) + NotOn []string `json:"not-on,omitempty"` // package to the tagged version a build pins Packages map[string]string `json:"packages,omitempty"` // a run: a program built from a change and run in the build seat's sandbox over the heads, its input asked of a verb of the asker's seat; the Go seat's check stream alone carries it, as a compose @@ -845,13 +848,17 @@ type StepEnded struct { // the ask's id ID string `json:"id"` // the packages that passed at their digest, for the runner to keep - Kept []KeptResult `json:"kept,omitempty"` - Module string `json:"module,omitempty"` + Kept []KeptResult `json:"kept,omitempty"` + // for a declined step, what the node lacked of them (design 47 H2 H-A-445) + Lacks []string `json:"lacks,omitempty"` + Module string `json:"module,omitempty"` + // for a declined step, the seats the package needs the node to hold (design 47 H2 H-A-445) + Needs []string `json:"needs,omitempty"` // the node that ran it, as in the subject Node string `json:"node"` // the tests that did not run, each with its reason, counted apart from passed and failed (design 47 H2 §800) NotRun []NotRun `json:"not-run,omitempty"` - // never passed for a step that ran nothing + // never passed for a step that ran nothing; declined for a step the holder's node lacks a need of, with needs and lacks, which the runner puts again on a node that has it (design 47 H2 H-A-375, H-A-445) Outcome string `json:"outcome"` // the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) Output string `json:"output,omitempty"` @@ -872,6 +879,7 @@ const ( StepEndedOutcomeFailed = "failed" StepEndedOutcomeNothingToRun = "nothing to run" StepEndedOutcomeCouldNotRun = "could not run" + StepEndedOutcomeDeclined = "declined" ) // The values of StepEnded's step. diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go index 7a73d14f6..dc17437a1 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go @@ -17,13 +17,13 @@ const Seat = "node-build-node" const Form = 1 // FormBuildStep is the form of what the part build-step adds. -const FormBuildStep = 2 +const FormBuildStep = 3 // The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition // whose fields changed while its forms did not (design 47 H2 §780). const ( - generatedForms = "1 build-step:2" - generatedFieldsDigest = "AnswerAnswer=a3b42bd5e791 AskArtifact=35eb85c27e97 AskTaken=8c917075eed4 AskTakenEvent=e4fc9b81e066 BuiltArtifact=b5fcef7a8b58 BuiltArtifactResolvedItem=3bcbbcfd1a1f Capacity=b40cca7ccdfa CheckAccept=ef89c6962ae4 CheckStepAccept=ef89c6962ae4 CurrentAnswer=e39824e012cb CurrentAsk=ac167e5ee11f DeliveryAccept=ef89c6962ae4 DeliveryStepAccept=ef89c6962ae4 DetectedStep=a6fcd7d0eca4 DetectedSteps=05198185c776 DetectedStepsByModuleArtifactValue=cb35b5775cc6 Drop=fb4637978e86 DropAnswer=7538f2f5e68e FailingTest=f3fb3f1a3ca8 Fetch=6e12677bbc2c FetchAnswer=46fc3d5bd7ee FetchBesideItem=774b4bf88ba4 FetchBuildsItem=aee5adc97246 FetchPackage=20d1668a0eee FetchRefusal=de1518848b4c InstallAccept=ef89c6962ae4 InstallAsk=9f965aa5b30b KeptArtifactAnswer=706729f8ccb9 KeptResult=8657683cdcd7 LevelAnswer=143c69c28a6a LevelAnswerDigest=d54b7fd7b1da LevelAnswerDigestNeedsItem=946b37d1d4a1 LevelAnswerDigestPackagesItem=a7d30558470d LevelAnswerKeptItem=85d3c2f78632 LevelAnswerUnmetItem=946b37d1d4a1 ModuleTree=00bb59e9f262 NotRun=d322fb1eb153 OutputAnswer=97ba4bd92bf9 Pause=d6521e0cdd1e PauseAnswer=7538f2f5e68e PausesByCapacityNodeValue=5392cf505c80 PausesByDroppedIDValue=0e15ac9cacbd PausesByPausedNodeValue=d111aad0f0cf PlacedBeside=53479337ff18 Program=b48ae74f34e0 RecheckAccept=ef89c6962ae4 ResumeAnswer=7538f2f5e68e RunHead=774b4bf88ba4 RunInput=3dc41505a4b7 RunJob=88af2a7f432b RunProgram=6fd636999d54 Step=75190430d510 StepAsk=a22b1f717fc3 StepAskBuild=de784b6c05da StepAskCheck=b77941df6dea StepAskCompose=4c6a08e10231 StepEnded=14b484f0ffa2 StepEndedEvent=207787118dac StepProgress=d25781cc895a StepProgressEvent=179344ce0ca2 StepResult=67f88be36d0e StepRun=a7a81f25515f StepsAnswer=2c02cac2df23 StepsByAskerNodeIDValue=50ccfd4cebd8 StepsByNodeIDValue=50ccfd4cebd8 TakenVersion=1b8d0547f61e TestCounts=9c1332427159 Toolchain=8ecb7debf9ad ToolchainReported=d26392767c50 ToolchainReportedEvent=fa0a431c9c7b ToolchainUsed=6cc7ccabc9f8 ToolchainsByNodeLanguageValue=f97d1dd27613" + generatedForms = "1 build-step:3" + generatedFieldsDigest = "AnswerAnswer=a3b42bd5e791 AskArtifact=35eb85c27e97 AskTaken=8c917075eed4 AskTakenEvent=e4fc9b81e066 BuiltArtifact=b5fcef7a8b58 BuiltArtifactResolvedItem=3bcbbcfd1a1f Capacity=b40cca7ccdfa CheckAccept=ef89c6962ae4 CheckStepAccept=ef89c6962ae4 CurrentAnswer=e39824e012cb CurrentAsk=ac167e5ee11f DeliveryAccept=ef89c6962ae4 DeliveryStepAccept=ef89c6962ae4 DetectedStep=a6fcd7d0eca4 DetectedSteps=05198185c776 DetectedStepsByModuleArtifactValue=cb35b5775cc6 Drop=fb4637978e86 DropAnswer=7538f2f5e68e FailingTest=f3fb3f1a3ca8 Fetch=6e12677bbc2c FetchAnswer=46fc3d5bd7ee FetchBesideItem=774b4bf88ba4 FetchBuildsItem=aee5adc97246 FetchPackage=20d1668a0eee FetchRefusal=de1518848b4c InstallAccept=ef89c6962ae4 InstallAsk=9f965aa5b30b KeptArtifactAnswer=706729f8ccb9 KeptResult=8657683cdcd7 LevelAnswer=143c69c28a6a LevelAnswerDigest=d54b7fd7b1da LevelAnswerDigestNeedsItem=946b37d1d4a1 LevelAnswerDigestPackagesItem=a7d30558470d LevelAnswerKeptItem=85d3c2f78632 LevelAnswerUnmetItem=946b37d1d4a1 ModuleTree=00bb59e9f262 NotRun=d322fb1eb153 OutputAnswer=97ba4bd92bf9 Pause=d6521e0cdd1e PauseAnswer=7538f2f5e68e PausesByCapacityNodeValue=5392cf505c80 PausesByDroppedIDValue=0e15ac9cacbd PausesByPausedNodeValue=d111aad0f0cf PlacedBeside=53479337ff18 Program=b48ae74f34e0 RecheckAccept=ef89c6962ae4 ResumeAnswer=7538f2f5e68e RunHead=774b4bf88ba4 RunInput=3dc41505a4b7 RunJob=88af2a7f432b RunProgram=6fd636999d54 Step=75190430d510 StepAsk=f0d09ea1670e StepAskBuild=de784b6c05da StepAskCheck=b77941df6dea StepAskCompose=4c6a08e10231 StepEnded=9bc24593a47d StepEndedEvent=207787118dac StepProgress=d25781cc895a StepProgressEvent=179344ce0ca2 StepResult=67f88be36d0e StepRun=a7a81f25515f StepsAnswer=2c02cac2df23 StepsByAskerNodeIDValue=50ccfd4cebd8 StepsByNodeIDValue=50ccfd4cebd8 TakenVersion=1b8d0547f61e TestCounts=9c1332427159 Toolchain=8ecb7debf9ad ToolchainReported=d26392767c50 ToolchainReportedEvent=fa0a431c9c7b ToolchainUsed=6cc7ccabc9f8 ToolchainsByNodeLanguageValue=f97d1dd27613" ) const ( @@ -810,8 +810,11 @@ type StepAsk struct { // build, check, compose, step or install Kind string `json:"kind"` // go, typescript or image - Language string `json:"language"` - Needs []string `json:"needs,omitempty"` + Language string `json:"language"` + // what the ask needs of the holder's node, by seat name: a check's packages' needs, and a pipeline step's, read from its artifact's check-needs in the module's manifest; a holder whose node lacks one declines the ask (ADR 0552 sections 2, 4; design 47 H2 H-A-375) + Needs []string `json:"needs,omitempty"` + // the nodes that declined the step before, to which the runner puts it again under a new id: a holder on a listed node hands it back at once, unacknowledged, for another node to take (design 47 H2 H-A-445) + NotOn []string `json:"not-on,omitempty"` // package to the tagged version a build pins Packages map[string]string `json:"packages,omitempty"` // a run: a program built from a change and run in the build seat's sandbox over the heads, its input asked of a verb of the asker's seat; the Go seat's check stream alone carries it, as a compose @@ -849,13 +852,17 @@ type StepEnded struct { // the ask's id ID string `json:"id"` // the packages that passed at their digest, for the runner to keep - Kept []KeptResult `json:"kept,omitempty"` - Module string `json:"module,omitempty"` + Kept []KeptResult `json:"kept,omitempty"` + // for a declined step, what the node lacked of them (design 47 H2 H-A-445) + Lacks []string `json:"lacks,omitempty"` + Module string `json:"module,omitempty"` + // for a declined step, the seats the package needs the node to hold (design 47 H2 H-A-445) + Needs []string `json:"needs,omitempty"` // the node that ran it, as in the subject Node string `json:"node"` // the tests that did not run, each with its reason, counted apart from passed and failed (design 47 H2 §800) NotRun []NotRun `json:"not-run,omitempty"` - // never passed for a step that ran nothing + // never passed for a step that ran nothing; declined for a step the holder's node lacks a need of, with needs and lacks, which the runner puts again on a node that has it (design 47 H2 H-A-375, H-A-445) Outcome string `json:"outcome"` // the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) Output string `json:"output,omitempty"` @@ -876,6 +883,7 @@ const ( StepEndedOutcomeFailed = "failed" StepEndedOutcomeNothingToRun = "nothing to run" StepEndedOutcomeCouldNotRun = "could not run" + StepEndedOutcomeDeclined = "declined" ) // The values of StepEnded's step. diff --git a/modules/mesh-delivery-contracts/parts/build-step.json b/modules/mesh-delivery-contracts/parts/build-step.json index a93030f8f..401d04597 100644 --- a/modules/mesh-delivery-contracts/parts/build-step.json +++ b/modules/mesh-delivery-contracts/parts/build-step.json @@ -502,7 +502,15 @@ "type": "array", "items": { "type": "string" - } + }, + "description": "what the ask needs of the holder's node, by seat name: a check's packages' needs, and a pipeline step's, read from its artifact's check-needs in the module's manifest; a holder whose node lacks one declines the ask (ADR 0552 sections 2, 4; design 47 H2 H-A-375)" + }, + "not-on": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the nodes that declined the step before, to which the runner puts it again under a new id: a holder on a listed node hands it back at once, unacknowledged, for another node to take (design 47 H2 H-A-445)" }, "packages": { "type": "object", @@ -1718,9 +1726,10 @@ "passed", "failed", "nothing to run", - "could not run" + "could not run", + "declined" ], - "description": "never passed for a step that ran nothing" + "description": "never passed for a step that ran nothing; declined for a step the holder's node lacks a need of, with needs and lacks, which the runner puts again on a node that has it (design 47 H2 H-A-375, H-A-445)" }, "tests": { "$ref": "#/defs/TestCounts" @@ -1743,6 +1752,20 @@ "type": "string", "description": "why it failed or could not run, naming the part not heard" }, + "needs": { + "type": "array", + "items": { + "type": "string" + }, + "description": "for a declined step, the seats the package needs the node to hold (design 47 H2 H-A-445)" + }, + "lacks": { + "type": "array", + "items": { + "type": "string" + }, + "description": "for a declined step, what the node lacked of them (design 47 H2 H-A-445)" + }, "output": { "type": "string", "description": "the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254)" @@ -2076,5 +2099,5 @@ } } ], - "form": 2 + "form": 3 } diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index f7cf601e7..0a05b98ba 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -1224,6 +1224,13 @@ "type": "string", "format": "date-time", "description": "when the seat's holder took the ask, from its ask-taken; the unanswered bound counts from it (design 47 H2 §778)" + }, + "not-on": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the nodes that declined the step's ask, lacking what it needs: its ask is put again under a new id naming them, each node declining at most once (design 47 H2 H-A-375, H-A-445)" } }, "description": "one step of a pipeline: its status in the policy and its outcome (design 47 *Pipeline steps and the policy*)", @@ -2203,5 +2210,5 @@ ] } }, - "form": 4 + "form": 5 } diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index cc7302e72..02eb103dd 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -510,6 +510,8 @@ export interface StepResult { "module"?: string; /** the node whose holder ended it */ "node"?: string; + /** the nodes that declined the step's ask, lacking what it needs: its ask is put again under a new id naming them, each node declining at most once (design 47 H2 H-A-375, H-A-445) */ + "not-on"?: string[]; /** for a waiting step, who or what it waits on: a person, a build node, the revision view */ "on"?: string; /** never passed for a step that ran nothing (design 47 *Pipeline steps and the policy*) */ diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index 079836868..d6a8207e0 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -450,7 +450,10 @@ export interface StepAsk { "kind": string; /** go, typescript or image */ "language": string; + /** what the ask needs of the holder's node, by seat name: a check's packages' needs, and a pipeline step's, read from its artifact's check-needs in the module's manifest; a holder whose node lacks one declines the ask (ADR 0552 sections 2, 4; design 47 H2 H-A-375) */ "needs"?: string[]; + /** the nodes that declined the step before, to which the runner puts it again under a new id: a holder on a listed node hands it back at once, unacknowledged, for another node to take (design 47 H2 H-A-445) */ + "not-on"?: string[]; /** package to the tagged version a build pins */ "packages"?: Record; /** a run: a program built from a change and run in the build seat's sandbox over the heads, its input asked of a verb of the asker's seat; the Go seat's check stream alone carries it, as a compose */ @@ -489,13 +492,17 @@ export interface StepEnded { "id": string; /** the packages that passed at their digest, for the runner to keep */ "kept"?: KeptResult[]; + /** for a declined step, what the node lacked of them (design 47 H2 H-A-445) */ + "lacks"?: string[]; "module"?: string; + /** for a declined step, the seats the package needs the node to hold (design 47 H2 H-A-445) */ + "needs"?: string[]; /** the node that ran it, as in the subject */ "node": string; /** the tests that did not run, each with its reason, counted apart from passed and failed (design 47 H2 §800) */ "not-run"?: NotRun[]; - /** never passed for a step that ran nothing */ - "outcome": "passed" | "failed" | "nothing to run" | "could not run"; + /** never passed for a step that ran nothing; declined for a step the holder's node lacks a need of, with needs and lacks, which the runner puts again on a node that has it (design 47 H2 H-A-375, H-A-445) */ + "outcome": "passed" | "failed" | "nothing to run" | "could not run" | "declined"; /** the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) */ "output"?: string; "pipeline"?: string; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index 079836868..d6a8207e0 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -450,7 +450,10 @@ export interface StepAsk { "kind": string; /** go, typescript or image */ "language": string; + /** what the ask needs of the holder's node, by seat name: a check's packages' needs, and a pipeline step's, read from its artifact's check-needs in the module's manifest; a holder whose node lacks one declines the ask (ADR 0552 sections 2, 4; design 47 H2 H-A-375) */ "needs"?: string[]; + /** the nodes that declined the step before, to which the runner puts it again under a new id: a holder on a listed node hands it back at once, unacknowledged, for another node to take (design 47 H2 H-A-445) */ + "not-on"?: string[]; /** package to the tagged version a build pins */ "packages"?: Record; /** a run: a program built from a change and run in the build seat's sandbox over the heads, its input asked of a verb of the asker's seat; the Go seat's check stream alone carries it, as a compose */ @@ -489,13 +492,17 @@ export interface StepEnded { "id": string; /** the packages that passed at their digest, for the runner to keep */ "kept"?: KeptResult[]; + /** for a declined step, what the node lacked of them (design 47 H2 H-A-445) */ + "lacks"?: string[]; "module"?: string; + /** for a declined step, the seats the package needs the node to hold (design 47 H2 H-A-445) */ + "needs"?: string[]; /** the node that ran it, as in the subject */ "node": string; /** the tests that did not run, each with its reason, counted apart from passed and failed (design 47 H2 §800) */ "not-run"?: NotRun[]; - /** never passed for a step that ran nothing */ - "outcome": "passed" | "failed" | "nothing to run" | "could not run"; + /** never passed for a step that ran nothing; declined for a step the holder's node lacks a need of, with needs and lacks, which the runner puts again on a node that has it (design 47 H2 H-A-375, H-A-445) */ + "outcome": "passed" | "failed" | "nothing to run" | "could not run" | "declined"; /** the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) */ "output"?: string; "pipeline"?: string; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index 079836868..d6a8207e0 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -450,7 +450,10 @@ export interface StepAsk { "kind": string; /** go, typescript or image */ "language": string; + /** what the ask needs of the holder's node, by seat name: a check's packages' needs, and a pipeline step's, read from its artifact's check-needs in the module's manifest; a holder whose node lacks one declines the ask (ADR 0552 sections 2, 4; design 47 H2 H-A-375) */ "needs"?: string[]; + /** the nodes that declined the step before, to which the runner puts it again under a new id: a holder on a listed node hands it back at once, unacknowledged, for another node to take (design 47 H2 H-A-445) */ + "not-on"?: string[]; /** package to the tagged version a build pins */ "packages"?: Record; /** a run: a program built from a change and run in the build seat's sandbox over the heads, its input asked of a verb of the asker's seat; the Go seat's check stream alone carries it, as a compose */ @@ -489,13 +492,17 @@ export interface StepEnded { "id": string; /** the packages that passed at their digest, for the runner to keep */ "kept"?: KeptResult[]; + /** for a declined step, what the node lacked of them (design 47 H2 H-A-445) */ + "lacks"?: string[]; "module"?: string; + /** for a declined step, the seats the package needs the node to hold (design 47 H2 H-A-445) */ + "needs"?: string[]; /** the node that ran it, as in the subject */ "node": string; /** the tests that did not run, each with its reason, counted apart from passed and failed (design 47 H2 §800) */ "not-run"?: NotRun[]; - /** never passed for a step that ran nothing */ - "outcome": "passed" | "failed" | "nothing to run" | "could not run"; + /** never passed for a step that ran nothing; declined for a step the holder's node lacks a need of, with needs and lacks, which the runner puts again on a node that has it (design 47 H2 H-A-375, H-A-445) */ + "outcome": "passed" | "failed" | "nothing to run" | "could not run" | "declined"; /** the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) */ "output"?: string; "pipeline"?: string; diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index a256a00f5..c177afd98 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -349,6 +349,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // since an ask's expired bound asks it once more through them (H-A-235). verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls) verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill} + // A declined step is put again on a node that has its need (design 47 H2 H-A-375, H-A-445). + hearer.NeedNodes, hearer.AskDeclined, hearer.PutAgain = rv.Holders.Nodes, verbs.DeclinedAsk, starter.PutAgain dropExpired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) askExpired := askBoundExpired(verbs.AskUnanswered, logger.Printf) expired := func(key string) { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index 440e2c3da..2c30941fc 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -94,6 +94,23 @@ func (h *holderView) Holders(seatName string) ([]string, bool) { return seatlist.HoldersIn(func() (map[string]json.RawMessage, error) { return pic.Follower().Values(), nil })(seatName), true } +// Nodes are the nodes on which the followed view names a holder of every seat of seats, sorted, and whether its first view +// is read (design 47 H2 H-A-445): a node seat's holders are keyed .. +func (h *holderView) Nodes(seats []string) ([]string, bool) { + h.mu.Lock() + pic := h.pic + h.mu.Unlock() + if pic == nil { + return nil, false + } + select { + case <-pic.Follower().Ready(): + default: + return nil, false + } + return pipeline.NodesHolding(pic.Follower().Values(), seats), true +} + // startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or // after, then start hub once. func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/declined.go b/modules/mesh-delivery-pipeline/internal/pipeline/declined.go new file mode 100644 index 000000000..ea3fbd385 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/declined.go @@ -0,0 +1,136 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "time" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" +) + +// A declined step (design 47 H2 H-A-375, H-A-445; ADR 0552 §2, §4): a build seat's holder whose node lacks what a step's +// packages need says its end declined, with the needs and what the node lacks. The step rides one shared queue, so the +// runner puts it again under a new ask id naming in not-on every node that declined it, each node declining at most once; +// the nodes that can take it are the holders of the step's seat and of every seat a need names (the seat-holders view) +// less those that declined. When none is left the step ends could not run, naming the need and the nodes asked. + +// declinedAgain is the ask that puts step i of p, declined on node as ev says, again on a node that has its needs, written +// into the step (its new ask, asked-at, not-on; taken-at and node cleared); nil and the why of its could-not-run end when +// no node is left or it cannot be asked, the step's not-on then naming node too. +func (h *Hearer) declinedAgain(p *seat.Pipeline, i int, ev StepEnded, node string, now time.Time) (*Asked, string) { + s := &p.Steps[i] + notOn := s.NotOn + if !slices.Contains(notOn, node) { + notOn = append(slices.Clone(notOn), node) + } + need := strings.Join(ev.Needs, ", ") + if need == "" { + need = strings.Join(ev.Lacks, ", ") + } + why := fmt.Sprintf("it needs %s, and no node that has it is left to ask: asked of %s, which declined it", need, strings.Join(notOn, ", ")) + var left []string + switch nodes, read := h.needNodes(append([]string{s.Seat}, ev.Needs...)); { + case !read: + why = fmt.Sprintf("it needs %s, declined by %s, and which nodes have it is not known: the seat-holders view is not read", need, strings.Join(notOn, ", ")) + default: + left = slices.DeleteFunc(nodes, func(n string) bool { return slices.Contains(notOn, n) }) + } + if len(left) == 0 || h.AskDeclined == nil || h.PutAgain == nil { + s.NotOn = notOn + return nil, why + } + again := *s + again.NotOn = notOn + a, err := h.AskDeclined(*p, again, now) + if err != nil { + s.NotOn = notOn + return nil, fmt.Sprintf("it needs %s, declined by %s, and it cannot be asked again: %v", need, strings.Join(notOn, ", "), err) + } + if b, ok := a.Ask.(nodebuildgo.StepAsk); ok { + b.NotOn = notOn + a.Ask = b + } + old := s.Ask + s.NotOn, s.Ask, s.AskedAt, s.TakenAt, s.Node = notOn, a.ID, &now, nil, "" + if run := sharedRun(*s); run != "" { + // A test run's ask: the run and every step of p reading it under the old ask name the new one (§725). + a.Run = run + for j := range p.Steps { + if sharedRun(p.Steps[j]) == run && p.Steps[j].Shared[0].Ask == old { + p.Steps[j].Shared[0].Ask = a.ID + } + } + s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode // until ask-taken (A-469b) + } + return a, "" +} + +// needNodes are the nodes holding every seat of seats, and whether the seat-holders view is read. +func (h *Hearer) needNodes(seats []string) ([]string, bool) { + if h.NeedNodes == nil { + return nil, false + } + return h.NeedNodes(seats) +} + +// putDeclined puts a declined step's new ask a: a test run's ask is first written into its run, so the run's end is taken +// from the new ask (§725); then the ask is put, its line in the journal. +func (h *Hearer) putDeclined(key, old string, s seat.StepResult, a Asked, ev StepEnded) error { + if a.Run != "" && h.TestRuns != nil { + for { + r, rev, err := h.TestRuns.Get(a.Run) + if errors.Is(err, ErrNoRun) || (err == nil && (r.Ended != nil || r.Ask != old)) { + break + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", a.Run, err) + } + r.Ask, r.Node = a.ID, "" + if _, err := h.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("the test run %s is not written: %w", a.Run, err) + } + break + } + } + h.logf("%s: the %s step of %s was declined on %s (%s): asked again as %s, not on %s (H-A-445)", key, s.Step, + strings.TrimSpace(s.Module+" "+s.Artifact), ev.Node, ev.Said, a.ID, strings.Join(s.NotOn, ", ")) + return h.PutAgain(key, a) +} + +// NodesHolding are the nodes on which values, the seat-holders view's keys and values, name a holder of every seat of seats, +// sorted: a node seat's holders are keyed . (design 47 H2 H-A-445). +func NodesHolding(values map[string]json.RawMessage, seats []string) []string { + seats = slices.Compact(slices.Sorted(slices.Values(seats))) + count := map[string]int{} + for key, raw := range values { + for _, s := range seats { + node, ok := strings.CutPrefix(key, s+".") + if !ok || node == "" || strings.Contains(node, ".") { + continue + } + var held struct { + Holders []struct { + Module string `json:"module"` + } `json:"holders"` + } + if json.Unmarshal(raw, &held) == nil && len(held.Holders) > 0 { + count[node]++ + } + } + } + var out []string + for node, n := range count { + if n == len(seats) { + out = append(out, node) + } + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index f9be05712..e74f96a53 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -3,6 +3,7 @@ package pipeline import ( "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "maps" @@ -20,6 +21,7 @@ import ( nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage" nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/seatdef" ) @@ -441,9 +443,37 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar } fetch.Packages[path] = version } - return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, + ask := nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint, Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}} + if step == seat.StepResultStepIntegration { + ask.Needs = checkNeedsOf(e, a.Ref) + } + return ask +} + +// checkNeedsOf are the seats artifact ref's integration tests need the node to hold, read from its check-needs in the +// module's manifest the entry keeps (design 47 H2 H-A-375, H-A-399 (3)): an integration step's ask names them, so a holder +// lacking one declines it. None when the manifest names none or cannot be read. +func checkNeedsOf(e catalog.CatalogEntry, ref string) []string { + if len(e.Manifest) == 0 { + return nil + } + raw, err := json.Marshal(e.Manifest) + if err != nil { + return nil + } + m, err := catalogue.Decode(raw) + if err != nil || m.Build == nil { + return nil + } + _, name, _ := strings.Cut(ref, ":") + for _, art := range m.Build.Artifacts { + if art.Name == name { + return art.CheckNeeds + } + } + return nil } // askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. arm, when not nil, arms diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index fe6a9153d..d62d8b1c1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -422,6 +422,18 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) { if tc, err := ToolchainOf([]byte(`{"language":"go","version":"1.26.1","digests":[]}`)); err != nil || tc.Language != "go" || tc.Version != "1.26.1" { t.Errorf("ToolchainOf = %+v, %v", tc, err) } + // Design 47 H2 H-A-375: an integration step's ask names its artifact's check-needs from the module's manifest; another + // step's names none. + e.Manifest = map[string]any{"module": "app", "version": "0.1.0", "build": map[string]any{"artifacts": []any{ + map[string]any{"name": "program", "kind": "program", "language": "go", "check-needs": []any{"node-build-node"}}}}} + m := seat.Mover{Module: "app", Commit: "c1", Fingerprint: "src2:aa"} + p := seat.Pipeline{Pipeline: "main.4", Branch: "main"} + if got := stepAsk(p, m, e, e.Artifacts[0], "go", seat.StepResultStepIntegration, "id", "the-runner", time.Now()).Needs; !slices.Equal(got, []string{"node-build-node"}) { + t.Errorf("the integration step's ask needs %v, want node-build-node", got) + } + if got := stepAsk(p, m, e, e.Artifacts[0], "go", seat.StepResultStepUnit, "id", "the-runner", time.Now()).Needs; got != nil { + t.Errorf("the unit step's ask needs %v, want none", got) + } } // An artifact without a language is placed by its kind (design 47 G §324, §397, A-324, A-325): an image is asked of the build @@ -668,6 +680,17 @@ func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) { } }) } + // Design 47 H2 H-A-445: the nodes that can take a declined step hold every seat named, as the view keys a node seat. + view := map[string]json.RawMessage{"node-build-go.ace": json.RawMessage(`{"holders":[{"module":"mesh-build-go"}]}`), + "node-build-go.g14": json.RawMessage(`{"holders":[{"module":"mesh-build-go"}]}`), "node-build-node.g14": json.RawMessage(`{"holders":[{"module":"mesh-build-node"}]}`), + "node-build-node.shanks": json.RawMessage(`{"holders":[{"module":"mesh-build-node"}]}`), "node-build-node.novox": json.RawMessage(`{"holders":[]}`), + "mesh-wants": json.RawMessage(`{"holders":[{"module":"assigner"}]}`)} + if got := NodesHolding(view, []string{"node-build-go", "node-build-node"}); !slices.Equal(got, []string{"g14"}) { + t.Errorf("the nodes holding both = %v, want g14", got) + } + if got := NodesHolding(view, []string{"node-build-node", "node-build-node"}); !slices.Equal(got, []string{"g14", "shanks"}) { + t.Errorf("the nodes holding node-build-node = %v, want g14 and shanks", got) + } claims := func(seats ...string) map[string]any { var cs []any for _, n := range seats { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 064c4ce2c..af7a28514 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -610,6 +610,16 @@ func (s *Starter) AskSteps(key string, asks []Asked) error { return s.askSteps(key, asks) } +// PutAgain puts one step's ask again under its new id, with its kept results and toolchain and its bound armed (a declined +// step's, design 47 H2 H-A-445): never joined to a test run, whose ask it is already. +func (s *Starter) PutAgain(key string, a Asked) error { + if s.Steps == nil { + return errors.New("the runner puts no ask") + } + s.carry(key, []Asked{a}) + return askAll(s.Steps, key, []Asked{a}, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm, s.logf) +} + // validateStep is the index of p's first validate step that names its ask, -1 when it has none. func validateStep(p seat.Pipeline) int { for i, st := range p.Steps { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index a4d3fea30..293616bed 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -187,6 +187,13 @@ type Hearer struct { // Unfinished keeps, at every end, the movers it leaves unfinished in the state unfinished (design 47 H2 §786); nil keeps // none. Unfinished *Unfinished + // NeedNodes are the nodes holding every seat of seats now, from the seat-holders view, and whether that view is read; a + // declined step is put again only on a node it names (design 47 H2 H-A-375, H-A-445). Nil knows none. + NeedNodes func(seats []string) ([]string, bool) + // AskDeclined is the ask of a declined step s of p under a new id (Verbs.DeclinedAsk), and PutAgain puts it on its seat's + // queue (Starter.PutAgain); with either nil a declined step ends could not run. + AskDeclined func(p seat.Pipeline, s seat.StepResult, now time.Time) (*Asked, error) + PutAgain func(key string, a Asked) error } // EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps @@ -290,6 +297,24 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh } now := h.now() s := &p.Steps[i] + if ev.Outcome == nodebuildgo.StepEndedOutcomeDeclined { + // Declined for a need the node lacks (design 47 H2 H-A-375, H-A-445): put again on a node that has it, or, none + // left, ended could not run naming the need and the nodes asked. + old := s.Ask + a, why := h.declinedAgain(&p, i, ev, node, now) + if a != nil { + if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err) + } + if err := h.putDeclined(ev.Pipeline, old, p.Steps[i], *a, ev); err != nil { + h.logf("%s: the declined ask %s is written as %s and not put, put again at the next start: %v", ev.Pipeline, old, a.ID, err) + } + return p, nil + } + ev.Outcome, ev.Said = nodebuildgo.StepEndedOutcomeCouldNotRun, why + } s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now if len(shared) > 0 { s.Shared = shared @@ -731,7 +756,7 @@ func phasesOf(p seat.Pipeline) *seat.PipelineEndedPhases { func endOutcome(o string) bool { switch o { case nodebuildgo.StepEndedOutcomePassed, nodebuildgo.StepEndedOutcomeFailed, nodebuildgo.StepEndedOutcomeNothingToRun, - nodebuildgo.StepEndedOutcomeCouldNotRun: + nodebuildgo.StepEndedOutcomeCouldNotRun, nodebuildgo.StepEndedOutcomeDeclined: return true } return false diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go index 481be1177..54af3a93d 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "testing" "time" @@ -174,6 +175,43 @@ func TestStepEndedEndsThePipeline(t *testing.T) { if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", unit); err != nil || len(*said) != n { t.Errorf("a second end said %v (%v)", (*said)[n:], err) } + + // Design 47 H2 H-A-375, H-A-445: a step declined for a need its node lacks is put again under a new ask naming the nodes + // that declined it, once per node; with no node left that has the need, it ends could not run naming the need and them. + t.Run("declined", func(t *testing.T) { + h, store, _, _, _ := fixture() + h.Asker = nil + var put []Asked + var seats []string + h.NeedNodes = func(s []string) ([]string, bool) { seats = s; return []string{"n1", "n2"}, true } + h.AskDeclined = func(p seat.Pipeline, s seat.StepResult, _ time.Time) (*Asked, error) { + id := s.Ask + "x" + return &Asked{Seat: s.Seat, ID: id, Ask: nodebuildgo.StepAsk{ID: id}}, nil + } + h.PutAgain = func(_ string, a Asked) error { put = append(put, a); return nil } + declined := func(ask, node string) StepEnded { + ev := ended(ask, "lint", nodebuildgo.StepEndedOutcomeDeclined) + ev.Node, ev.Needs, ev.Lacks, ev.Said = node, []string{"node-build-node"}, []string{"node-build-node"}, "this node lacks node-build-node" + return ev + } + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", declined("a1", "n1")); err != nil { + t.Fatal(err) + } + lint := store[pkey].p.Steps[0] + if lint.Outcome != seat.StepResultOutcomeRunning || lint.Ask != "a1x" || !slices.Equal(lint.NotOn, []string{"n1"}) || lint.AskedAt == nil || + len(put) != 1 || put[0].ID != "a1x" || !slices.Equal(put[0].Ask.(nodebuildgo.StepAsk).NotOn, []string{"n1"}) || + !slices.Equal(seats, []string{nodebuildgo.Seat, "node-build-node"}) { + t.Fatalf("after the first decline: the step %+v, put %+v, nodes asked for %v; want it running under a1x, not on n1", lint, put, seats) + } + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n2", declined("a1x", "n2")); err != nil { + t.Fatal(err) + } + lint = store[pkey].p.Steps[0] + if want := "it needs node-build-node, and no node that has it is left to ask: asked of n1, n2, which declined it"; lint.Outcome != seat.StepResultOutcomeCouldNotRun || + lint.Why != want || len(put) != 1 || lint.Ended == nil { + t.Errorf("after the second decline: the step %+v, %d puts; want could not run, %q", lint, len(put), want) + } + }) } // The approval step is asked last, only when every other required step passed, and ended by its warrant (§26, §37). diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 47e6ac144..147d49ba7 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -553,6 +553,21 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time return a, nil } +// DeclinedAsk is the ask of a declined step s of p under a new id (design 47 H2 H-A-445): the one Place makes for it, as a +// rerun's is (a kept mover's kept revision mixed in), naming s's not-on. +func (v Verbs) DeclinedAsk(p seat.Pipeline, s seat.StepResult, now time.Time) (*Asked, error) { + id := keptAskID(now, p.Pipeline, s.Module, s.Artifact, s.Step+" not on "+strings.Join(s.NotOn, ","), v.keptOf(p.Pipeline)[s.Module]) + a, err := v.placedAsk(p, s, id, now) + if err != nil { + return nil, err + } + if b, ok := a.Ask.(nodebuildgo.StepAsk); ok { + b.NotOn = s.NotOn + a.Ask = b + } + return a, nil +} + // placedAsk is the ask of step s under id: a validate or composition step of the seat mesh-wants built directly for its // mover (H-A-236), any other the one Place makes for it. func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) { diff --git a/modules/mesh-delivery-sdk/buildstep/ask.go b/modules/mesh-delivery-sdk/buildstep/ask.go index 23ca15d49..042ffcee5 100644 --- a/modules/mesh-delivery-sdk/buildstep/ask.go +++ b/modules/mesh-delivery-sdk/buildstep/ask.go @@ -296,8 +296,13 @@ type StepAsk struct { // H-A-403): the module's npm tree its package lock names, published as the module's archive :node-modules // under the ask's Version. Install *InstallAsk `json:"install,omitempty"` - // Needs are what a check's tests need of the holder's node (CheckNeed); a holder lacking one declines the ask. + // Needs are what a check's or a pipeline step's tests need of the holder's node, by seat name (CheckNeed): a step's are + // read from its artifact's check-needs in the module's manifest (design 47 H2 H-A-375); a holder lacking one declines + // the ask. Needs []string `json:"needs,omitempty"` + // NotOn are the nodes that declined the step before (design 47 H2 H-A-445): a holder on a listed node hands the ask + // back at once, unacknowledged, for another node to take. + NotOn []string `json:"not-on,omitempty"` // Packages pin the mesh packages a build or a check builds against. Packages PackageVersions `json:"packages,omitempty"` } @@ -510,8 +515,8 @@ func (a StepAsk) check() error { return fmt.Errorf("buildstep: the ask %s is a check on the delivery stream, which carries trunk builds alone", a.ID) case a.Check != nil && a.Language == AskImage: return fmt.Errorf("buildstep: the ask %s is a check of an image, which has no tests", a.ID) - case len(a.Needs) > 0 && a.Check == nil: - return fmt.Errorf("buildstep: the ask %s names needs and is no check, whose tests alone need a node", a.ID) + case len(a.Needs) > 0 && a.Check == nil && a.Step == nil: + return fmt.Errorf("buildstep: the ask %s names needs and is no check or pipeline step, whose tests alone need a node", a.ID) case len(a.Packages) > 0 && a.Compose != nil: return fmt.Errorf("buildstep: the ask %s is a compose and names package versions, which a compose does not take", a.ID) } diff --git a/modules/mesh-delivery-sdk/buildstep/pipelinestep.go b/modules/mesh-delivery-sdk/buildstep/pipelinestep.go index 6c7413153..275340aab 100644 --- a/modules/mesh-delivery-sdk/buildstep/pipelinestep.go +++ b/modules/mesh-delivery-sdk/buildstep/pipelinestep.go @@ -34,10 +34,13 @@ const ( OutcomeFailed = "failed" OutcomeNothingToRun = "nothing to run" OutcomeCouldNotRun = "could not run" + // OutcomeDeclined is a pipeline step the holder's node lacks a need of, said with Needs and Lacks: the runner puts it + // again on a node that has the need (design 47 H2 H-A-375, H-A-445). + OutcomeDeclined = "declined" ) // StepOutcomes are the outcomes a build seat may say. -var StepOutcomes = []string{OutcomePassed, OutcomeFailed, OutcomeNothingToRun, OutcomeCouldNotRun} +var StepOutcomes = []string{OutcomePassed, OutcomeFailed, OutcomeNothingToRun, OutcomeCouldNotRun, OutcomeDeclined} // StepEndedVersion is the layout version of a step's end. const StepEndedVersion = 1 @@ -125,24 +128,28 @@ func (r StepRun) check(id string) error { // §602): a pipeline step's end names its pipeline, step, module, artifact, commit and fingerprint; the end of any other // ask names its asker, the ask, the node, the outcome and when, and neither commit nor fingerprint (G §548). type StepEnded struct { - V int `json:"v"` - ID string `json:"id"` - Asker string `json:"asker"` - Ask string `json:"ask"` - Pipeline string `json:"pipeline,omitempty"` - Step string `json:"step,omitempty"` - Module string `json:"module,omitempty"` - Artifact string `json:"artifact,omitempty"` - Commit string `json:"commit,omitempty"` - Fingerprint string `json:"fingerprint,omitempty"` - Node string `json:"node"` - Outcome string `json:"outcome"` - Tests *TestCounts `json:"tests,omitempty"` - Carried []KeptResult `json:"carried,omitempty"` - Kept []KeptResult `json:"kept,omitempty"` - Said string `json:"said,omitempty"` - Output string `json:"output,omitempty"` - AnswerObject string `json:"answer-object,omitempty"` + V int `json:"v"` + ID string `json:"id"` + Asker string `json:"asker"` + Ask string `json:"ask"` + Pipeline string `json:"pipeline,omitempty"` + Step string `json:"step,omitempty"` + Module string `json:"module,omitempty"` + Artifact string `json:"artifact,omitempty"` + Commit string `json:"commit,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + Node string `json:"node"` + Outcome string `json:"outcome"` + Tests *TestCounts `json:"tests,omitempty"` + Carried []KeptResult `json:"carried,omitempty"` + Kept []KeptResult `json:"kept,omitempty"` + Said string `json:"said,omitempty"` + // Needs and Lacks are, for a declined step, the seats the package needs the node to hold and what the node lacked of + // them (design 47 H2 H-A-445). + Needs []string `json:"needs,omitempty"` + Lacks []string `json:"lacks,omitempty"` + Output string `json:"output,omitempty"` + AnswerObject string `json:"answer-object,omitempty"` // NotRun are the tests that did not run, each with its reason, counted apart from passed and failed (design 47 H2 // §800). NotRun []NotRun `json:"not-run,omitempty"` @@ -286,7 +293,10 @@ func EncodeStepEnded(e StepEnded) ([]byte, error) { if err := e.check(); err != nil { return nil, err } - if (e.Outcome == OutcomeCouldNotRun || e.Outcome == OutcomeFailed) && e.Said == "" { + if e.Outcome == OutcomeDeclined && (e.Pipeline == "" || len(e.Lacks) == 0) { + return nil, fmt.Errorf("buildstep: the end of %s says declined and is no pipeline step's or names nothing its node lacks", e.ID) + } + if (e.Outcome == OutcomeCouldNotRun || e.Outcome == OutcomeFailed || e.Outcome == OutcomeDeclined) && e.Said == "" { return nil, fmt.Errorf("buildstep: the end of %s says %s and not why", e.ID, e.Outcome) } return json.Marshal(e) diff --git a/modules/mesh-delivery-sdk/steprun/askers.go b/modules/mesh-delivery-sdk/steprun/askers.go index 76eaf2a0f..878405866 100644 --- a/modules/mesh-delivery-sdk/steprun/askers.go +++ b/modules/mesh-delivery-sdk/steprun/askers.go @@ -75,6 +75,26 @@ func (p *Puller) sayAskEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said str } } +// sayDeclined says the end of an ask this node declined: a pipeline step's as the outcome declined with what it needs and +// what this node lacks, for the runner to put it again on a node that has it (design 47 H2 H-A-375, H-A-445); any other +// ask's as could not run, for its asker to put again (issue 890). +func (p *Puller) sayDeclined(msg *Pulled, ask sdkstep.StepAsk, needs, lacks []string, said string) { + if ask.Step == nil { + p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, said) + return + } + end, err := sdkstep.EndOf(ask, p.Node, sdkstep.OutcomeDeclined) + if err != nil { + return + } + end.Said, end.At, end.Asker, end.Needs, end.Lacks = said, now().UTC(), askerOf(msg, ask), needs, lacks + if body, err := sdkstep.EncodeStepEnded(end); err == nil { + p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID) + } else { + p.logf("the decline of the ask %s on %s is not said: %v", ask.ID, p.Node, err) + } +} + // PriorRuns is Puller.Prior over a steps state's read (a module's ReadSteps.Get): the Run of the step a key holds. Nil for no // read (a test's), so the bus's delivery count stands. func PriorRuns(get func(key string) (json.RawMessage, error)) func(string) int { diff --git a/modules/mesh-delivery-sdk/steprun/askrun_test.go b/modules/mesh-delivery-sdk/steprun/askrun_test.go index 5201560d8..3ebd439ac 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun_test.go +++ b/modules/mesh-delivery-sdk/steprun/askrun_test.go @@ -466,7 +466,12 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { arch string // the archive the end names text string // the run's output kept as . (design 47 H2 §791), "" for none checked failing []sdkstep.FailingTest + needs []sdkstep.PackageNeeds // the digest's packages' needs, which this node (Has nil) lacks + declined bool // the run declines, naming the needs and what this node lacks, and says nothing }{ + // Design 47 H2 H-A-375, H-A-445: a step whose package needs what this node lacks is declined, never could not run. + {name: "integration needing node-build-node on a node lacking it: declined", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{"x/b": "d-b"}, + tests: map[string]bool{"x/b": true}, needs: []sdkstep.PackageNeeds{{Package: "x/b", Needs: []string{sdkstep.NeedNodeBuildNode}}}, declined: true}, {name: "unit: the kept package carried, the other tested and kept", ask: stepOf(sdkstep.StepUnit, sdkstep.KeptResult{Package: "x/a", Digest: "d-a", Level: "unit"}), digests: map[string]string{"x/a": "d-a", "x/b": "d-b"}, tests: map[string]bool{"x/b": true}, outcome: sdkstep.OutcomePassed, counts: 1, carried: []string{"x/a"}, kept: []string{"x/b"}}, {name: "unit failed: said with why, nothing kept", ask: stepOf(sdkstep.StepUnit), digests: map[string]string{"x/b": "d-b"}, tests: map[string]bool{"x/b": false}, @@ -546,7 +551,15 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { got.ends = append(got.ends, e) return nil } + r.needs = row.needs out, err := RunAsk(ctx, s, row.ask) + if row.declined { + var dec *Declining + if !errors.As(err, &dec) || !slices.Equal(dec.Needs, []string{sdkstep.NeedNodeBuildNode}) || !slices.Equal(dec.Lacks, []string{sdkstep.NeedNodeBuildNode}) || len(got.ends) != 0 { + t.Errorf("%s: %v, said %+v; want it declined, needing and lacking node-build-node, nothing said", row.name, err, got.ends) + } + continue + } if row.passing { if !sdkstep.IsPassing(err) || len(got.ends) != 0 { t.Errorf("%s: %v, said %+v; want a transport fault, nothing said", row.name, err, got.ends) diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index aa755bbb0..f5b3b8dd8 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sort" "strings" "sync" @@ -620,7 +621,7 @@ func (p *Puller) decline(msg *Pulled, ask sdkstep.StepAsk, lacks []string) { return } p.logf("the ask %s is declined on %s: it needs %s, and this node lacks %s", ask.ID, p.Node, strings.Join(ask.Needs, ", "), strings.Join(lacks, ", ")) - p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, "this node lacks "+strings.Join(lacks, ", ")) + p.sayDeclined(msg, ask, ask.Needs, lacks, "this node lacks "+strings.Join(lacks, ", ")) _ = msg.Ack() // A-489a: its end is written p.mu.Lock() if p.declined == nil { @@ -630,6 +631,20 @@ func (p *Puller) decline(msg *Pulled, ask sdkstep.StepAsk, lacks []string) { p.mu.Unlock() } +// handBack hands back, unacknowledged and at once, a step ask whose not-on names this node, which declined it before +// (design 47 H2 H-A-445): the queue delivers it on to another node, and the worker it came from is left alone until the next +// pull's wait, as after a decline, so this node does not take it again at once. +func (p *Puller) handBack(msg *Pulled, ask sdkstep.StepAsk) { + p.logf("the ask %s is handed back on %s: it names this node among those that declined it (%s)", ask.ID, p.Node, strings.Join(ask.NotOn, ", ")) + _ = msg.Nak() + p.mu.Lock() + if p.declined == nil { + p.declined = map[string]bool{} + } + p.declined[msg.Kind] = true + p.mu.Unlock() +} + // declineRun ends the step of an ask that began and then found this node should not run it (Declining), as decline does // before it begins: refused as declined, the message dropped for its asker to put again, and the worker it came from left // alone until the next pull's wait. @@ -756,6 +771,10 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.refuseDropped(msg, ask, d) return } + if slices.Contains(ask.NotOn, p.Node) { + p.handBack(msg, ask) + return + } if lacks := sdkstep.Lacks(ask.Needs, p.Has); len(lacks) > 0 { p.decline(msg, ask, lacks) return @@ -854,7 +873,11 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key var dec *Declining if errors.As(err, &dec) && ctx.Err() == nil { p.declineRun(msg, ask, key, step, dec) - p.sayAskEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text) + needs := dec.Needs + if len(needs) == 0 { + needs = ask.Needs + } + p.sayDeclined(msg, ask, needs, dec.Lacks, dec.Text) return } if ctx.Err() != nil { diff --git a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go index 67e8728e6..3017a1663 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_poison_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_poison_test.go @@ -136,9 +136,39 @@ func TestAnEndThatCannotBeWrittenIsGivenBack(t *testing.T) { } } -// A-489a: a decline at the take (a need the node lacks), a refused run (Declining) and a dropped ask each say their pipeline step's end as could not -// run, naming its pipeline, and are acknowledged once it is written. +// A-489a: a decline at the take (a need the node lacks), a refused run (Declining) and a dropped ask each say their pipeline step's end, +// naming its pipeline, and are acknowledged once it is written: a dropped one as could not run, a declined one as declined with what +// it needs and what this node lacks (design 47 H2 H-A-375, H-A-445). A step ask whose not-on names this node is handed back at once, +// unacknowledged and unsaid. func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) { + t.Run("not on this node", func(t *testing.T) { + src, st := NewFakeAsks(), newSteps() + a := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(), + NotOn: []string{"ace", "shanks"}, Needs: []string{sdkstep.NeedNodeBuildNode}, + Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: sdkstep.StepIntegration, Module: "m", Artifact: "m:program", Commit: strings.Repeat("a", 40), + Fingerprint: "fp-1", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: strings.Repeat("a", 40), Dir: "modules/m"}}} + raw, err := sdkstep.EncodeStepAsk(a) + if err != nil { + t.Fatal(err) + } + src.Publish(sdkstep.AskCheck, raw) + said := 0 + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, PullWait: 10 * time.Millisecond, + Has: func(string) bool { return true }, + Say: func(string, any, string) error { said++; return nil }, + Run: func(context.Context, sdkstep.StepAsk) (any, error) { + t.Error("an ask not on this node ran") + return nil, nil + }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "the ask handed back", func() bool { _, _, nk := src.Fates(); return len(nk) >= 1 }) + cancel() + if acked, termed, _ := src.Fates(); len(acked)+len(termed) != 0 || said != 0 { + t.Errorf("acked %v termed %v said %d, want it handed back alone", acked, termed, said) + } + }) for _, c := range []string{"declined", "declined in its run", "dropped"} { t.Run(c, func(t *testing.T) { src, st := NewFakeAsks(), newSteps() @@ -178,8 +208,7 @@ func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) { } } if c == "declined" { - // The decoder refuses needs on a pipeline step ask, so no ask of the queue reaches the decline; the line is held - // by calling it with the ask built whole. + a.Needs = []string{sdkstep.NeedNodeBuildNode} acked := 0 p.decline(&Pulled{Ack: func() error { acked++; return nil }, Term: func() error { return nil }, Nak: func() error { return nil }}, a, []string{"node-build-node"}) if acked != 1 { @@ -196,8 +225,15 @@ func TestAnEndedPipelineStepIsSaidAndAcknowledged(t *testing.T) { mu.Lock() e := ended[0] mu.Unlock() - if e.Outcome != sdkstep.OutcomeCouldNotRun || e.Pipeline != "main.12.m" { - t.Errorf("the end said %+v, want could not run naming its pipeline", e) + want, needs, lacks := sdkstep.OutcomeDeclined, "node-build-node", "node-build-node" + switch c { + case "dropped": + want, needs, lacks = sdkstep.OutcomeCouldNotRun, "", "" + case "declined in its run": + needs, lacks = "", "the toolchain" + } + if e.Outcome != want || e.Pipeline != "main.12.m" || strings.Join(e.Needs, ",") != needs || strings.Join(e.Lacks, ",") != lacks { + t.Errorf("the end said %+v, want %s naming its pipeline, needs %q and lacks %q", e, want, needs, lacks) } if _, termed, naked := src.Fates(); len(termed)+len(naked) != 0 { t.Errorf("termed %v naked %v, want it acknowledged alone", termed, naked) diff --git a/modules/mesh-delivery-sdk/steprun/refusal.go b/modules/mesh-delivery-sdk/steprun/refusal.go index fb9bfffe9..6e7cd4204 100644 --- a/modules/mesh-delivery-sdk/steprun/refusal.go +++ b/modules/mesh-delivery-sdk/steprun/refusal.go @@ -24,6 +24,9 @@ func Refuse(format string, args ...any) *Refusal { return &Refusal{Text: fmt.Spr type Declining struct { Text string Lacks []string // what this node lacks, in words the operator reads + // Needs are, for a pipeline step, the seats its packages need the node to hold (design 47 H2 H-A-375, H-A-445): its end + // says declined with them and Lacks. + Needs []string } func (d *Declining) Error() string { return d.Text } diff --git a/modules/mesh-delivery-sdk/steprun/steprun.go b/modules/mesh-delivery-sdk/steprun/steprun.go index 1b3eebf66..ac8029b0f 100644 --- a/modules/mesh-delivery-sdk/steprun/steprun.go +++ b/modules/mesh-delivery-sdk/steprun/steprun.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "time" @@ -181,11 +182,19 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste } la := answer.Levels[0] if len(la.Unmet) > 0 { - var what []string + // The node lacks what the packages' tests need: the step is declined with the needs and what this node lacks of them, + // for the runner to put it again on a node that has them (design 47 H2 H-A-375, H-A-445). + var what, needs []string for _, u := range la.Unmet { what = append(what, u.Package+" ("+strings.Join(u.Needs, ", ")+")") + for _, n := range u.Needs { + if !slices.Contains(needs, n) { + needs = append(needs, n) + } + } } - return fmt.Errorf("the %s tests of %s need what this node lacks", level, strings.Join(what, ", ")) + return &Declining{Text: fmt.Sprintf("the %s tests of %s need what this node lacks", level, strings.Join(what, ", ")), + Needs: needs, Lacks: sdkstep.Lacks(needs, s.Has)} } tests, passed, ran := 0, true, false var why []string