A pipeline's kept revisions are pinned; a lost one ends the pipeline loudly; the held take-in chain is visible (design 47 H2 H-A-496, H-A-497) #338
@@ -14,13 +14,13 @@ const Seat = "mesh-pipeline-runner"
|
||||
|
||||
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
|
||||
// a reader of a value of another form says "form N, this reader knows M".
|
||||
const Form = 8
|
||||
const Form = 9
|
||||
|
||||
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
|
||||
// whose fields changed while its forms did not (design 47 H2 §780).
|
||||
const (
|
||||
generatedForms = "8"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionAgainNValue=693580eed498 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
generatedForms = "9"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=582f8ff3b28d PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionAgainNValue=693580eed498 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -948,7 +948,7 @@ const (
|
||||
PipelineRowOutcomeCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// PipelineStarted is a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52).
|
||||
// PipelineStarted is a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52), and again at a re-pin with its kept revisions (H2 H-A-496).
|
||||
type PipelineStarted struct {
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
// the watched branch, or the branch of a run on request
|
||||
@@ -957,6 +957,8 @@ type PipelineStarted struct {
|
||||
Call string `json:"call,omitempty"`
|
||||
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
|
||||
Form int64 `json:"form,omitempty"`
|
||||
// every catalog revision the pipeline's movers are kept at, said again in full at each re-pin, so the indexer pins each while the pipeline runs (design 47 H2 H-A-496 and its addendum)
|
||||
Kept []int64 `json:"kept,omitempty"`
|
||||
// the pipeline's movers
|
||||
Movers []Mover `json:"movers"`
|
||||
// the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)
|
||||
|
||||
@@ -19,6 +19,8 @@ func TestFixtures(t *testing.T) {
|
||||
}{
|
||||
{name: "a take-in's pipeline started with its two movers", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.3589", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"repository\":\"mesh-sdk\",\"tier\":1},{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":2}],\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"repository\":\"mesh-sdk\",\"tier\":1},{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":2}],\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineStartedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-started\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline started again at a re-pin, with the three revisions its movers are kept at (H-A-496)", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.291", given: "{\"at\":\"2026-11-22T20:00:00Z\",\"branch\":\"main\",\"kept\":[292,295,298],\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"pipeline\":\"main.291\",\"revision\":291,\"source\":114,\"v\":1}", wire: "{\"at\":\"2026-11-22T20:00:00Z\",\"branch\":\"main\",\"kept\":[292,295,298],\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"pipeline\":\"main.291\",\"revision\":291,\"source\":114,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineStartedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-started\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline ended passed: per mover its version, steps and tier, an optional step failed", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.3589", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v PipelineEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.pipeline-ended\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline of two movers ended with its phases and one mover left unfinished", subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.3589", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"form\":2,\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"phases\":{\"build\":{\"end\":\"2026-11-21T22:02:00Z\",\"start\":\"2026-11-21T22:00:06Z\"},\"composition\":{\"end\":\"2026-11-21T22:06:00Z\",\"start\":\"2026-11-21T22:04:01Z\"},\"ended\":\"2026-11-21T22:06:00Z\",\"placed\":\"2026-11-21T22:00:05Z\",\"taken\":\"2026-11-21T22:00:00Z\",\"unit\":{\"end\":\"2026-11-21T22:04:00Z\",\"start\":\"2026-11-21T22:02:01Z\"}},\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"unfinished\":[\"mesh-delivery-view\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"form\":2,\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-sdk\",\"module\":\"mesh-sdk\",\"steps\":[{\"module\":\"mesh-sdk\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"validate\"}],\"tier\":1,\"version\":\"0.1.21\",\"widens\":[{\"name\":\"seat:mesh-wants.input\",\"what\":\"grant\"}]},{\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"tier\":2,\"version\":\"0.1.15\"}],\"outcome\":\"passed\",\"phases\":{\"build\":{\"end\":\"2026-11-21T22:02:00Z\",\"start\":\"2026-11-21T22:00:06Z\"},\"composition\":{\"end\":\"2026-11-21T22:06:00Z\",\"start\":\"2026-11-21T22:04:01Z\"},\"ended\":\"2026-11-21T22:06:00Z\",\"placed\":\"2026-11-21T22:00:05Z\",\"taken\":\"2026-11-21T22:00:00Z\",\"unit\":{\"end\":\"2026-11-21T22:04:00Z\",\"start\":\"2026-11-21T22:02:01Z\"}},\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"unfinished\":[\"mesh-delivery-view\"],\"v\":1}",
|
||||
|
||||
@@ -29,6 +29,29 @@
|
||||
"at": "2026-11-10T10:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a pipeline started again at a re-pin, with the three revisions its movers are kept at (H-A-496)",
|
||||
"of": "event:pipeline-started",
|
||||
"subject": "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.291",
|
||||
"given": {
|
||||
"pipeline": "main.291",
|
||||
"branch": "main",
|
||||
"revision": 291,
|
||||
"source": 114,
|
||||
"v": 1,
|
||||
"movers": [
|
||||
{
|
||||
"module": "nats",
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"fingerprint": "fp-0123",
|
||||
"repository": "mesh-networking",
|
||||
"tier": 1
|
||||
}
|
||||
],
|
||||
"kept": [292, 295, 298],
|
||||
"at": "2026-11-22T20:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a pipeline ended passed: per mover its version, steps and tier, an optional step failed",
|
||||
"of": "event:pipeline-ended",
|
||||
|
||||
@@ -1675,9 +1675,16 @@
|
||||
"published": {
|
||||
"$ref": "#/defs/Published",
|
||||
"description": "for a pipeline a package's publish started, that publish (A-202)"
|
||||
},
|
||||
"kept": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
},
|
||||
"description": "every catalog revision the pipeline's movers are kept at, said again in full at each re-pin, so the indexer pins each while the pipeline runs (design 47 H2 H-A-496 and its addendum)"
|
||||
}
|
||||
},
|
||||
"description": "a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52)",
|
||||
"description": "a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52), and again at a re-pin with its kept revisions (H2 H-A-496)",
|
||||
"required": [
|
||||
"v",
|
||||
"pipeline",
|
||||
@@ -2367,5 +2374,5 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"form": 8
|
||||
"form": 9
|
||||
}
|
||||
|
||||
@@ -446,7 +446,7 @@ export interface PipelineRow {
|
||||
"steps"?: Record<string, number>;
|
||||
}
|
||||
|
||||
/** a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52) */
|
||||
/** a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52), and again at a re-pin with its kept revisions (H2 H-A-496) */
|
||||
export interface PipelineStarted {
|
||||
"at"?: string;
|
||||
/** the watched branch, or the branch of a run on request */
|
||||
@@ -455,6 +455,8 @@ export interface PipelineStarted {
|
||||
"call"?: string;
|
||||
/** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */
|
||||
"form"?: number;
|
||||
/** every catalog revision the pipeline's movers are kept at, said again in full at each re-pin, so the indexer pins each while the pipeline runs (design 47 H2 H-A-496 and its addendum) */
|
||||
"kept"?: number[];
|
||||
/** the pipeline's movers */
|
||||
"movers": Mover[];
|
||||
/** the pipeline's key, <branch>.<revision>, or <branch>.<source>.published.<package>.<version> for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */
|
||||
|
||||
@@ -664,7 +664,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
sources.refresh()
|
||||
})
|
||||
// The held take-ins are taken in again from the state held at the start, before a branch's consumer gives a later one
|
||||
// (design 47 H, H-A-114, H-A-118).
|
||||
// (design 47 H, H-A-114, H-A-118); a chain's key a stop left is deleted first (H2 H-A-497 addendum).
|
||||
ix.EndChains(ctx)
|
||||
sources.takeAgain()
|
||||
sources.refresh()
|
||||
if stopped != nil {
|
||||
|
||||
@@ -265,24 +265,34 @@ type actor struct {
|
||||
s *sourceMoved
|
||||
}
|
||||
|
||||
// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending).
|
||||
// HeldAgain takes a branch's held source revisions in again now (indexer_take_in_pending), beside the asks' loop, and answers
|
||||
// at once (design 47 H2 H-A-497 and its addendum): a chain that runs already answers its state from the state takeins; held
|
||||
// sources start the chain at the first, written there before it runs, answered with how many wait.
|
||||
func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn {
|
||||
r := index.TakenIn{Module: branch}
|
||||
if hs, err := a.HeldOf(ctx, branch); err != nil || len(hs) == 0 {
|
||||
if c, running, err := a.ChainOf(ctx, branch); err != nil {
|
||||
r.Answer = "the state takeins is not read: " + err.Error()
|
||||
return r
|
||||
} else if running {
|
||||
r.Held = true
|
||||
r.Answer = fmt.Sprintf("a chain runs: source %d taken in since %s, %d wait", c.Source, c.Since.Format(time.RFC3339), c.Waiting)
|
||||
return r
|
||||
}
|
||||
hs, err := a.HeldOf(ctx, branch)
|
||||
if err != nil || len(hs) == 0 {
|
||||
r.Answer = "the branch " + branch + " holds no source-moved"
|
||||
if err != nil {
|
||||
r.Answer = "the state held is not read: " + err.Error()
|
||||
}
|
||||
return r
|
||||
}
|
||||
a.s.takeAgain()
|
||||
hs, err := a.HeldOf(ctx, branch)
|
||||
still := err != nil || len(hs) > 0
|
||||
r.Stored, r.Held = !still, still
|
||||
r.Answer = "taken in"
|
||||
if still {
|
||||
r.Answer = "still held: indexer_pending says why"
|
||||
}
|
||||
c := a.StartChain(ctx, hs)
|
||||
go func() {
|
||||
a.s.takeAgain()
|
||||
a.EndChain(a.s.ctx, branch)
|
||||
}()
|
||||
r.Held = true
|
||||
r.Answer = fmt.Sprintf("chain started at source %d, %d wait", c.Source, c.Waiting)
|
||||
return r
|
||||
}
|
||||
|
||||
|
||||
@@ -97,6 +97,98 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// chainStore is a takeins state in memory, safe beside the chain's own goroutine, which says on ended when a chain key goes.
|
||||
type chainStore struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
ended chan string
|
||||
}
|
||||
|
||||
func (c *chainStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
v, ok := c.vals[k]
|
||||
return v, 1, ok, nil
|
||||
}
|
||||
func (c *chainStore) Keys(context.Context) ([]string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var out []string
|
||||
for k := range c.vals {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (c *chainStore) Create(_ context.Context, k string, v []byte) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.vals[k] = v
|
||||
return nil
|
||||
}
|
||||
func (c *chainStore) Update(ctx context.Context, k string, v []byte, _ uint64) error {
|
||||
return c.Create(ctx, k, v)
|
||||
}
|
||||
func (c *chainStore) Delete(_ context.Context, k string, _ uint64) error {
|
||||
c.mu.Lock()
|
||||
_, had := c.vals[k]
|
||||
delete(c.vals, k)
|
||||
c.mu.Unlock()
|
||||
if had && strings.HasPrefix(k, index.ChainPrefix) {
|
||||
c.ended <- k
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-497 and its addendum: a hand ask of a branch with held sources starts the chain beside the asks' loop
|
||||
// and answers at once; an ask that meets the running chain answers its state, which the state takeins holds and
|
||||
// indexer_pending shows; the chain's key goes at its end.
|
||||
func TestAHandAskOfAHeldBranchAnswersAtOnce(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
held := &memStore{vals: map[string][]byte{}}
|
||||
takeIns := &chainStore{vals: map[string][]byte{}, ended: make(chan string, 4)}
|
||||
ix := &index.Indexer{Held: held, TakeIns: takeIns, Logf: func(string, ...any) {},
|
||||
Now: func() time.Time { return time.Date(2026, 11, 22, 21, 7, 0, 0, time.UTC) }}
|
||||
handlers := map[string]func(jetstream.Msg){}
|
||||
s := &sourceMoved{ctx: ctx, ix: ix,
|
||||
branches: func(context.Context) ([]string, error) { return []string{"main"}, nil },
|
||||
bind: func(_ context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
|
||||
handlers[branch] = each
|
||||
return func() {}, nil
|
||||
},
|
||||
logf: func(string, ...any) {}}
|
||||
s.refresh()
|
||||
for _, n := range []int{114, 115, 116} {
|
||||
m := &fakeMsg{subject: fmt.Sprintf("mesh.seat.mesh-delivery.event.source-moved.main.%d", n),
|
||||
data: []byte(fmt.Sprintf(`{"v":1,"branch":"main","source":%d,"call":"%d123456789abcde","heads":[]}`, n, n))}
|
||||
handlers["main"](msgOf{f: m})
|
||||
}
|
||||
a := actor{ix, s}
|
||||
s.again.Lock() // the chain waits for the lock, so it is still running at the second ask
|
||||
if r := a.HeldAgain(ctx, "main"); r.Answer != "chain started at source 114, 2 wait" {
|
||||
s.again.Unlock()
|
||||
t.Fatalf("the first ask answered %+v, want the chain started at once", r)
|
||||
}
|
||||
r := a.HeldAgain(ctx, "main")
|
||||
var pending any
|
||||
var err error
|
||||
for _, tool := range index.HeldTools(held, takeIns, nil, ix.Now, nil) {
|
||||
if tool.Name == index.PendingTool {
|
||||
pending, err = tool.Run(map[string]any{})
|
||||
}
|
||||
}
|
||||
s.again.Unlock()
|
||||
chains := pending.(index.PendingAnswer).Chains
|
||||
if r.Answer != "a chain runs: source 114 taken in since 2026-11-22T21:07:00Z, 2 wait" || !r.Held {
|
||||
t.Errorf("the ask meeting the chain answered %+v, want its state at once", r)
|
||||
}
|
||||
if err != nil || len(chains) != 1 || chains[0].Branch != "main" || chains[0].Source != 114 || chains[0].Waiting != 2 {
|
||||
t.Errorf("indexer_pending shows chains %+v (%v), want main's at 114 with 2 waiting", chains, err)
|
||||
}
|
||||
if k := <-takeIns.ended; k != index.ChainKey("main") {
|
||||
t.Errorf("the key %s went, want main's chain", k)
|
||||
}
|
||||
}
|
||||
|
||||
// revisionObjects is the catalog revisions' object bucket in memory.
|
||||
type revisionObjects map[string][]byte
|
||||
|
||||
|
||||
@@ -33,8 +33,8 @@ type takerIn interface {
|
||||
// written back, by compare-and-set at its revision, with what the indexer stored or why not. A value
|
||||
// answered already (this indexer's own write) or deleted is left.
|
||||
func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) {
|
||||
return // a running take-in's waiting-on is the indexer's own record, no ask (H-A-409)
|
||||
if e.Operation() != jetstream.KeyValuePut || strings.HasPrefix(e.Key(), index.RunningPrefix) || strings.HasPrefix(e.Key(), index.ChainPrefix) {
|
||||
return // a running take-in's waiting-on and a running chain are the indexer's own records, no ask (H-A-409, H-A-497)
|
||||
}
|
||||
a, err := index.DecodeTakeIn(e.Value())
|
||||
if err != nil {
|
||||
|
||||
@@ -51,6 +51,9 @@ type PendingAnswer struct {
|
||||
Held []HeldTakeIn `json:"held"`
|
||||
// Running are the take-ins that run, each with what it waits on (design 47 H2 §803, H-A-409).
|
||||
Running []RunningTakeIn `json:"running,omitempty"`
|
||||
// Chains are the running chains of held take-ins, each branch's source being taken in, since when and how many wait
|
||||
// (H-A-497 addendum).
|
||||
Chains []HeldChain `json:"chains,omitempty"`
|
||||
}
|
||||
|
||||
// ErrHeldUnwritten says a source-moved that could not be taken in is not written into the state held either: its message
|
||||
@@ -167,26 +170,153 @@ func (ix *Indexer) unhold(ctx context.Context, branch string, source int64) {
|
||||
}
|
||||
}
|
||||
|
||||
// TakeHeldIn takes every branch's held source revisions in, each branch in its order, on the indexer's own room and at its
|
||||
// start (H-A-114, H-A-118): a branch stops at the first that still cannot be taken in, which stays held with those behind it.
|
||||
// TakeHeldIn takes every branch's held source revisions in, each branch in its order, on the indexer's own room, at its
|
||||
// start and by hand (H-A-114, H-A-118): a branch stops at the first that still cannot be taken in, which stays held with those
|
||||
// behind it. Once a held one is taken in, the next held of its branch is taken in at once, each such start said in the journal
|
||||
// (design 47 H2 H-A-497).
|
||||
func (ix *Indexer) TakeHeldIn(ctx context.Context) {
|
||||
all, err := ReadHeld(ctx, ix.Held)
|
||||
if err != nil {
|
||||
ix.logf("take-in: the state %s is not read, so no held take-in is taken in again now: %v", HeldState, err)
|
||||
return
|
||||
}
|
||||
stopped := map[string]bool{}
|
||||
stopped, landed := map[string]bool{}, map[string]int64{}
|
||||
behind := map[string]int{}
|
||||
for _, h := range all {
|
||||
behind[h.Branch]++
|
||||
}
|
||||
for i, h := range all {
|
||||
behind[h.Branch]--
|
||||
if stopped[h.Branch] {
|
||||
continue
|
||||
}
|
||||
if _, err := ix.TakeInSource(ctx, h.Event()); err != nil {
|
||||
if prev, ok := landed[h.Branch]; ok {
|
||||
ix.logf("take-in: held source %d of %s taken in next, after %d (H-A-497)", h.Source, h.Branch, prev)
|
||||
}
|
||||
ix.writeChain(ctx, HeldChain{Branch: h.Branch, Source: h.Source, Since: ix.now().UTC(), Waiting: behind[h.Branch]})
|
||||
if _, err := ix.TakeInSource(ctx, h.Event()); err == nil {
|
||||
landed[h.Branch] = h.Source
|
||||
} else {
|
||||
ix.logf("take-in: source revision %d of %s is still held: %v", h.Source, h.Branch, err)
|
||||
stopped[h.Branch] = true
|
||||
}
|
||||
// The chain of a branch ends at its first held again or its last held (H-A-497 addendum).
|
||||
if stopped[h.Branch] || !slices.ContainsFunc(all[i+1:], func(n HeldTakeIn) bool { return n.Branch == h.Branch }) {
|
||||
ix.endChain(ctx, h.Branch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ChainPrefix is the takeins keys of the chains of held take-ins: `chain.<branch>`, the branch escaped as a key's name is,
|
||||
// holding the branch's chain while it runs (design 47 H2 H-A-497 and its addendum): written at each step, deleted at its end.
|
||||
const ChainPrefix = "chain."
|
||||
|
||||
// ChainKey is the takeins key of branch's chain.
|
||||
func ChainKey(branch string) string { return ChainPrefix + keys.EscapeName(branch) }
|
||||
|
||||
// HeldChain is a running chain of a branch's held take-ins (H-A-497 addendum): the source being taken in, since when, and
|
||||
// how many held sources of the branch wait behind it.
|
||||
type HeldChain struct {
|
||||
Branch string `json:"branch"`
|
||||
Source int64 `json:"source"`
|
||||
Since time.Time `json:"since"`
|
||||
Waiting int `json:"waiting"`
|
||||
}
|
||||
|
||||
// writeChain writes branch's chain at its step c into the state takeins; a failed write is said.
|
||||
func (ix *Indexer) writeChain(ctx context.Context, c HeldChain) {
|
||||
if ix.TakeIns == nil {
|
||||
return
|
||||
}
|
||||
data, err := json.Marshal(c)
|
||||
if err == nil {
|
||||
err = write(ctx, ix.TakeIns, ChainKey(c.Branch), func([]byte, bool) ([]byte, bool, error) { return data, false, nil })
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("take-in: the chain of %s at source revision %d is not written into %s: %v", c.Branch, c.Source, TakeInsState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// EndChain deletes branch's chain from the state takeins once the take-in its ask started returned, so a chain whose held
|
||||
// sources another act took first is never left running.
|
||||
func (ix *Indexer) EndChain(ctx context.Context, branch string) { ix.endChain(ctx, branch) }
|
||||
|
||||
// EndChains deletes every chain the state takeins holds, at the indexer's start (H-A-497 addendum): no chain runs before the
|
||||
// start's take-in, so a key left by a stop between a chain's last take-in and its end would answer "a chain runs" for good.
|
||||
func (ix *Indexer) EndChains(ctx context.Context) {
|
||||
chains, err := ReadChains(ctx, ix.TakeIns)
|
||||
if err != nil {
|
||||
ix.logf("take-in: the chains in %s are not read, so none left by a stop is deleted: %v", TakeInsState, err)
|
||||
return
|
||||
}
|
||||
for _, c := range chains {
|
||||
ix.logf("take-in: the chain of %s at source revision %d, left by a stop, is deleted at the start", c.Branch, c.Source)
|
||||
ix.endChain(ctx, c.Branch)
|
||||
}
|
||||
}
|
||||
|
||||
// endChain deletes branch's chain from the state takeins.
|
||||
func (ix *Indexer) endChain(ctx context.Context, branch string) {
|
||||
if ix.TakeIns == nil {
|
||||
return
|
||||
}
|
||||
if err := write(ctx, ix.TakeIns, ChainKey(branch), func([]byte, bool) ([]byte, bool, error) { return nil, true, nil }); err != nil {
|
||||
ix.logf("take-in: the chain of %s is not deleted from %s: %v", branch, TakeInsState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// StartChain writes branch's chain at its first held source, before the chain is taken in beside the asks (H-A-497
|
||||
// addendum), so an ask after it finds it running; it answers that chain.
|
||||
func (ix *Indexer) StartChain(ctx context.Context, held []HeldTakeIn) HeldChain {
|
||||
c := HeldChain{Branch: held[0].Branch, Source: held[0].Source, Since: ix.now().UTC(), Waiting: len(held) - 1}
|
||||
ix.writeChain(ctx, c)
|
||||
return c
|
||||
}
|
||||
|
||||
// ChainOf is branch's running chain in the state takeins, and whether one runs.
|
||||
func (ix *Indexer) ChainOf(ctx context.Context, branch string) (HeldChain, bool, error) {
|
||||
if ix.TakeIns == nil {
|
||||
return HeldChain{}, false, nil
|
||||
}
|
||||
v, _, found, err := ix.TakeIns.Get(ctx, ChainKey(branch))
|
||||
if err != nil || !found {
|
||||
return HeldChain{}, false, err
|
||||
}
|
||||
var c HeldChain
|
||||
if err := json.Unmarshal(v, &c); err != nil {
|
||||
return HeldChain{}, false, fmt.Errorf("%s.%s: %w", TakeInsState, ChainKey(branch), err)
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
|
||||
// ReadChains are the running chains the state takeins holds (H-A-497 addendum), each branch's once.
|
||||
func ReadChains(ctx context.Context, st Store) ([]HeldChain, error) {
|
||||
if st == nil {
|
||||
return nil, nil
|
||||
}
|
||||
ks, err := st.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(ks)
|
||||
var out []HeldChain
|
||||
for _, k := range ks {
|
||||
if !strings.HasPrefix(k, ChainPrefix) {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := st.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var c HeldChain
|
||||
if !found || json.Unmarshal(v, &c) != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SkipTakeIn ends a held source revision unwritten by hand (indexer_take_in_skip, §744): it says take-in.taken with no entries,
|
||||
// skipped, by and why, and deletes the branch's held take-in; the caller acknowledges the source-moved.
|
||||
func (ix *Indexer) SkipTakeIn(ctx context.Context, e meshdelivery.SourceMovedEvent, by, why string) error {
|
||||
@@ -236,7 +366,7 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs
|
||||
{Name: PendingTool,
|
||||
Description: "Each watched branch's held take-in (design 47 H §744): the source revision of the gate's source-moved the indexer " +
|
||||
"could not take in (the catalog store full, a read of the forge unanswered), why and since when, in order, each kept here once " +
|
||||
"acknowledged and taken in again on the indexer's room and at its start; and each running take-in that waits over a minute on another part, its waiting-on (part, verb, since). indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.",
|
||||
"acknowledged and taken in again on the indexer's room and at its start; each running take-in that waits over a minute on another part, its waiting-on (part, verb, since); and each running chain of held take-ins, the source being taken in, since when and how many wait. indexer_take_in_pending takes it again now, indexer_take_in_skip ends it unwritten.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
h, err := ReadHeld(context.Background(), held)
|
||||
@@ -244,11 +374,16 @@ func HeldTools(held, takeIns Store, ask func(key string, value []byte) (TakeInAs
|
||||
return PendingAnswer{Held: h}, err
|
||||
}
|
||||
r, err := ReadRunning(context.Background(), takeIns)
|
||||
return PendingAnswer{Held: h, Running: r}, err
|
||||
if err != nil {
|
||||
return PendingAnswer{Held: h, Running: r}, err
|
||||
}
|
||||
c, err := ReadChains(context.Background(), takeIns)
|
||||
return PendingAnswer{Held: h, Running: r, Chains: c}, err
|
||||
}},
|
||||
{Name: TakeInPendingTool,
|
||||
Description: "Take a watched branch's held source-moved in again now (design 47 H §744), as the indexer's room would: " +
|
||||
"answers whether it was taken in. indexer_pending names the held branches.",
|
||||
"the held sources are taken in in order beside the asks, and it answers at once that the chain started at a source and " +
|
||||
"how many wait, or, while a chain runs, its state (design 47 H2 H-A-497). indexer_pending names the held branches and the running chains.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"branch"}, "properties": map[string]any{
|
||||
"branch": map[string]any{"type": "string", "description": "the watched branch, as indexer_pending names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken again now"}}},
|
||||
|
||||
@@ -73,8 +73,8 @@ func pinObject(key string) (string, bool) {
|
||||
}
|
||||
|
||||
// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a
|
||||
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it and every
|
||||
// other revision the record pins (H-A-147). A body
|
||||
// pipeline-started pins the revision it names, and each revision its movers are kept at (design 47 H2 H-A-496), under the
|
||||
// record `pipeline.<pipeline>`, its pipeline-ended frees it and every other revision the record pins (H-A-147). A body
|
||||
// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it
|
||||
// again; one applied twice changes nothing.
|
||||
func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error {
|
||||
@@ -83,7 +83,15 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
}
|
||||
switch {
|
||||
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
||||
if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)); err != nil {
|
||||
record := PipelineRecord(started.Pipeline)
|
||||
if err := ix.pin(ctx, PinKey(started.Branch, started.Revision), record); err != nil {
|
||||
return err
|
||||
}
|
||||
// Every revision the pipeline's movers are kept at is pinned under its record too, so no judge removes one it still
|
||||
// reads (design 47 H2 H-A-496); the latest started's kept set is the pipeline's kept pins, so a kept revision the
|
||||
// record pins that this started no longer names is freed (H-A-496 addendum). A kept revision is always newer than the
|
||||
// pipeline's own (a re-pin of its own source, H-A-283), so its own revision and its validate base stay pinned.
|
||||
if err := ix.pinKept(ctx, started.Branch, started.Revision, started.Kept, record); err != nil {
|
||||
return err
|
||||
}
|
||||
// The reservations of the pipeline's revision settle at its start, if they gather (design 47 H2 §805, H-A-424 (1)).
|
||||
@@ -229,6 +237,41 @@ func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// pinKept pins each of kept, the revisions a pipeline of branch at revision own keeps movers at, under record, and frees
|
||||
// every revision newer than own the record pins that kept does not name (design 47 H2 H-A-496 and its addendum).
|
||||
func (ix *Indexer) pinKept(ctx context.Context, branch string, own int64, kept []int64, record string) error {
|
||||
for _, r := range kept {
|
||||
if r <= 0 || r == own {
|
||||
continue
|
||||
}
|
||||
if err := ix.pin(ctx, PinKey(branch, r), record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
b, r, ok := ParsePinKey(k)
|
||||
if !ok || b != branch || r <= own || slices.Contains(kept, r) {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pv PinValue
|
||||
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
|
||||
continue
|
||||
}
|
||||
if err := ix.unpin(ctx, k, record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PipelineRecord is the record a pipeline pins its revision under: `pipeline.<branch>.<source>` (design 47 H §743, A-449a).
|
||||
func PipelineRecord(pipeline string) string { return "pipeline." + pipeline }
|
||||
|
||||
|
||||
@@ -13,6 +13,47 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 H2 H-A-496 and its addendum: a pipeline-started pins the pipeline's own revision and each revision its movers
|
||||
// are kept at under the pipeline's record; a later started (said again at a re-pin) is the pipeline's kept set, freeing a
|
||||
// kept revision it no longer names but never the pipeline's own; pipeline-ended frees them all.
|
||||
func TestAPipelinesKeptRevisionsArePinnedUntilItEnds(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pending := uint64(1) // the keeps judge nothing while the consumer is behind
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
|
||||
Now: func() time.Time { return time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC) }, PinsPending: func(context.Context) (uint64, error) { return pending, nil }}
|
||||
kept := func() []string {
|
||||
n, _ := ix.Revisions.Names(ctx)
|
||||
return slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok })
|
||||
}
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||
for source := int64(1); source <= 6; source++ {
|
||||
e := reader
|
||||
e.Commit = fmt.Sprintf("c%d", source)
|
||||
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pending = 0
|
||||
heard := func(started *runner.PipelineStarted, ended *runner.PipelineEnded, want ...string) {
|
||||
t.Helper()
|
||||
if err := ix.HeardPins(ctx, started, ended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
if got := kept(); !slices.Equal(got, want) {
|
||||
t.Errorf("kept %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{2, 3, 4}}, nil,
|
||||
"main.1", "main.2", "main.3", "main.4", "main.6")
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, "main.3"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.1"}) {
|
||||
t.Errorf("the pin of main.3 holds %s, want the pipeline's record", v)
|
||||
}
|
||||
heard(&runner.PipelineStarted{Branch: "main", Pipeline: "main.1", Revision: 1, Kept: []int64{3, 4}}, nil,
|
||||
"main.1", "main.3", "main.4", "main.6")
|
||||
heard(nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.1", Revision: 1, Outcome: runner.PipelineEndedOutcomeFailed}, "main.6")
|
||||
}
|
||||
|
||||
// Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its
|
||||
// pipeline-ended, counts a record once, and removes a revision only while its consumers are caught up, judged against the pins
|
||||
// its own state holds, never a count of pins stated: one two pipelines pin stays until both free it, one neither pinned nor
|
||||
|
||||
@@ -371,11 +371,35 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
|
||||
len(held[0].Heads) != 1 || held[0].Heads[0] != moved.Heads[0] || !reflect.DeepEqual(held[0].Event(), moved) {
|
||||
t.Fatalf("held = %+v, want 3 then 4, each with its event whole", held)
|
||||
}
|
||||
// H-A-497: behind them a source whose head the forge does not answer, and one more behind it.
|
||||
for _, n := range []struct {
|
||||
source int64
|
||||
commit string
|
||||
}{{5, "c-unread"}, {6, f.Merge}} {
|
||||
e := moved
|
||||
e.Source, e.Call, e.Heads = n.source, fmt.Sprintf("%d123456789abcdef", n.source), []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: n.commit}}
|
||||
if behind, err := ix.HoldBehind(ctx, e); err != nil || !behind {
|
||||
t.Fatalf("source %d held behind: %v (%v)", n.source, behind, err)
|
||||
}
|
||||
}
|
||||
var lines []string
|
||||
ix.Logf = func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) }
|
||||
ix.TakeIns = newMem()
|
||||
// H-A-497 addendum: while the chain runs, its key names the source being taken in and how many wait behind it.
|
||||
var steps []string
|
||||
read := ix.Read
|
||||
ix.Read = func(ctx context.Context, repository, sha, dir string) (RepoRead, error) {
|
||||
if c, ok, _ := ix.ChainOf(ctx, "main"); ok {
|
||||
steps = append(steps, fmt.Sprintf("%d/%d", c.Source, c.Waiting))
|
||||
}
|
||||
return read(ctx, repository, sha, dir)
|
||||
}
|
||||
objects.capacity = 1 << 22
|
||||
if ok, err := ix.RoomAgain(ctx); err != nil || !ok || rooms != 1 {
|
||||
t.Fatalf("room said %v %d times (%v), want once the store has room", ok, rooms, err)
|
||||
}
|
||||
// H-A-114: on room (and at start) the held ones are taken in again from the state held alone, in their order.
|
||||
// H-A-114, H-A-497: on room (and at start, and by hand) the held ones are taken in again from the state held alone, in
|
||||
// their order, each next one at once and said in the journal, until one is held again: those behind it stay held.
|
||||
ix.TakeHeldIn(ctx)
|
||||
var taken []int64
|
||||
for _, e := range *said {
|
||||
@@ -386,8 +410,38 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
|
||||
if !reflect.DeepEqual(taken, []int64{3, 4}) {
|
||||
t.Errorf("taken again on room: %v of %+v, want 3 then 4", taken, *said)
|
||||
}
|
||||
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 {
|
||||
t.Errorf("held = %+v after the take-in, want none", held)
|
||||
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 2 || held[0].Source != 5 || held[1].Source != 6 {
|
||||
t.Errorf("held = %+v after the take-in, want 5 held again and 6 behind it", held)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"take-in: held source 4 of main taken in next, after 3 (H-A-497)",
|
||||
"take-in: held source 5 of main taken in next, after 4 (H-A-497)",
|
||||
} {
|
||||
if !slices.Contains(lines, want) {
|
||||
t.Errorf("the journal holds no line %q: %q", want, lines)
|
||||
}
|
||||
}
|
||||
for _, l := range lines {
|
||||
if strings.Contains(l, "held source 6 of main taken in") {
|
||||
t.Errorf("source 6 was taken in behind a held one: %q", l)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(steps, "4/2") || !slices.Contains(steps, "5/1") {
|
||||
t.Errorf("the chain's key read at each take-in %v, want main at 4 with 2 waiting and at 5 with 1", steps)
|
||||
}
|
||||
// H-A-497 addendum: the chain's key in the state takeins goes at its end, here its first held again, and one a stop
|
||||
// left is deleted at the start.
|
||||
if chains, err := ReadChains(ctx, ix.TakeIns); err != nil || len(chains) != 0 {
|
||||
t.Errorf("chains %+v (%v) after the chain stopped, want none", chains, err)
|
||||
}
|
||||
ix.writeChain(ctx, HeldChain{Branch: "main", Source: 9, Waiting: 1})
|
||||
if ix.EndChains(ctx); func() bool { c, _ := ReadChains(ctx, ix.TakeIns); return len(c) != 0 }() {
|
||||
t.Error("a chain's key left by a stop stays after the start")
|
||||
}
|
||||
ix.Read = read
|
||||
ix.Logf, ix.TakeIns = nil, nil
|
||||
for _, src := range []int64{5, 6} {
|
||||
ix.unhold(ctx, "main", src)
|
||||
}
|
||||
// A skip ends a held source revision unwritten: take-in.taken with no entries, skipped, by and why; held is deleted.
|
||||
ix.hold(ctx, meshdelivery.SourceMovedEvent{Branch: "nox-mesh", Source: 9, Call: "fedcba9876543210"}, "the catalog store is full")
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -150,10 +151,17 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi
|
||||
for {
|
||||
raw, err := call("seat:"+catalog.Seat+"."+catalog.VerbRevision, in)
|
||||
if err != nil {
|
||||
// A revision the catalog answers "no such object" for is gone (design 47 H2 H-A-496): a kept mover's wait on it ends.
|
||||
if strings.Contains(err.Error(), "no such object") {
|
||||
return pipeline.Half{}, fmt.Errorf("%w: %v", pipeline.ErrRevisionGone, err)
|
||||
}
|
||||
return pipeline.Half{}, err
|
||||
}
|
||||
answer, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
if strings.Contains(said, "no such object") {
|
||||
return pipeline.Half{}, fmt.Errorf("%w: %s answered: %s", pipeline.ErrRevisionGone, catalog.VerbRevision, said)
|
||||
}
|
||||
return pipeline.Half{}, fmt.Errorf("%s answered: %s", catalog.VerbRevision, said)
|
||||
}
|
||||
var a catalog.RevisionAnswer
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -63,6 +64,19 @@ func TestCatalogRevisionIsKeptOnceRead(t *testing.T) {
|
||||
if h, err := read("main", 0); err != nil || calls != 3 || len(h.Entries) != 2 {
|
||||
t.Fatalf("the newest made calls %d, %+v, %v; want its first page only", calls, h, err)
|
||||
}
|
||||
// A revision the catalog answers "no such object" for is gone, whether refused or answered so (design 47 H2 H-A-496).
|
||||
for _, gone := range []func(string, any) (json.RawMessage, error){
|
||||
func(string, any) (json.RawMessage, error) {
|
||||
return nil, fmt.Errorf("revision main.292: no such object")
|
||||
},
|
||||
func(string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"content":[{"text":"revision main.292: no such object"}],"isError":true}`), nil
|
||||
},
|
||||
} {
|
||||
if _, err := catalogRevision(gone, newRevisionCache(1))("main", 292); !errors.Is(err, pipeline.ErrRevisionGone) {
|
||||
t.Errorf("a revision the catalog has no object of reads %v, want it gone", err)
|
||||
}
|
||||
}
|
||||
c := newRevisionCache(1)
|
||||
c.put("main", pipeline.Half{Revision: 1})
|
||||
c.put("main", pipeline.Half{Revision: 2})
|
||||
|
||||
@@ -132,6 +132,11 @@ func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Move
|
||||
s.logf("%s keeps the movers revision %d of its own source %d re-pinned, %s: %d steps ended superseded, %d asked again (H-A-283, H-A-284)",
|
||||
key, half.Revision, half.Source, strings.Join(moved, ", "), len(ended), len(asks))
|
||||
s.sayRePinned(p, kept, revisions, half.Revision)
|
||||
// The re-pin says pipeline-started again with every revision its movers are kept at, so the indexer pins them (design
|
||||
// 47 H2 H-A-496 and its addendum).
|
||||
if keeps {
|
||||
s.sayStartedPipeline(p, now)
|
||||
}
|
||||
for _, st := range ended {
|
||||
if s.Say == nil {
|
||||
break
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
asks := &stepAsks{}
|
||||
var said []seat.StepEvent
|
||||
var repinned []string
|
||||
var repinned, starts []string
|
||||
var left [][]seat.StepResult
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: asks, Name: "the-runner", Now: ticking(), Revision: read,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
|
||||
@@ -77,6 +77,9 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
if ev, ok := body.(seat.RePinnedEvent); ok {
|
||||
repinned = append(repinned, fmt.Sprintf("%s %s %s %d", subject, ev.Pipeline, ev.Mover, ev.Revision))
|
||||
}
|
||||
if ev, ok := body.(seat.PipelineStarted); ok {
|
||||
starts = append(starts, fmt.Sprintf("%s %v", ev.Pipeline, ev.Kept))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Left: func(_ string, leave, drop []seat.StepResult, _ string) { left = append(left, append(leave, drop...)) }}
|
||||
@@ -133,6 +136,11 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
if kept, err := store.KeptRevisions(key); err != nil || kept["app"] != 11 {
|
||||
t.Errorf("kept-revisions = %v (%v), want app at 11", kept, err)
|
||||
}
|
||||
// design 47 H2 H-A-496 and its addendum: the start names no kept revision, the re-pin says pipeline-started again with
|
||||
// every revision its movers are kept at.
|
||||
if want := []string{key + " []", key + " [11]"}; !slices.Equal(starts, want) {
|
||||
t.Errorf("pipeline-started said %q, want %q", starts, want)
|
||||
}
|
||||
// The pin check reads the kept mover's entry at the revision it was kept at, never the pipeline's.
|
||||
if n, err := s.AskPinHeld(store, key, "app", map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26", "git.novox.be/novox/x/go": "v0.1.1"}, read); n == 0 || err != nil {
|
||||
after, _, _ := store.Get(key)
|
||||
@@ -255,6 +263,60 @@ func TestAKeptMoverWaitingOnItsRevisionSaysSo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// design 47 H2 H-A-496: a kept mover whose kept revision the catalog answers "no such object" for waits no longer: the
|
||||
// pipeline ends failed "kept revision 11 gone" at the placing, is said ended, frees its turn, and leaves its mover
|
||||
// unfinished, so it rejoins the next pipeline.
|
||||
func TestAKeptRevisionGoneEndsThePipeline(t *testing.T) {
|
||||
reg, entries, _, read := repinFixture()
|
||||
revision := func(branch string, rev int64) (Half, error) {
|
||||
if rev == 11 {
|
||||
return Half{}, fmt.Errorf("revision main.11: no such object: %w", ErrRevisionGone)
|
||||
}
|
||||
return read(branch, rev)
|
||||
}
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
var turned, ended []string
|
||||
u := memUnfinished{}
|
||||
h := &Hearer{Store: store, Name: "the-runner", Unfinished: &Unfinished{Store: u},
|
||||
Say: func(subject string, body any) error {
|
||||
if ev, ok := body.(seat.PipelineEnded); ok {
|
||||
ended = append(ended, ev.Pipeline+" "+ev.Outcome)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Turn: func(key string) { turned = append(turned, key) }}
|
||||
registered := false
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: &stepAsks{}, Name: "the-runner", Now: ticking(), Revision: revision,
|
||||
Say: func(string, any) error { return nil }, Ended: h.EndedAtPlacing,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
|
||||
key := Key("main", 10)
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: 41, Revision: 10, Entries: entries,
|
||||
Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
if n, err := s.Revised(revisionEvent(11), read); n != 0 || err != nil {
|
||||
t.Fatalf("the re-pin started %d (%v)", n, err)
|
||||
}
|
||||
registered = true
|
||||
lagging := func(string) (Half, bool) { return Half{Revision: 10, Source: 41, Entries: entries}, true }
|
||||
if _, err := s.Registered(store, reg, lagging); err != nil {
|
||||
t.Fatalf("the placing: %v", err)
|
||||
}
|
||||
p, _, err := store.Get(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Outcome != seat.PipelineOutcomeFailed || p.Ended == nil || !strings.HasPrefix(p.Why, "kept revision 11 gone") {
|
||||
t.Fatalf("the pipeline = %s, %q, ended %v: want failed with kept revision 11 gone", p.Outcome, p.Why, p.Ended)
|
||||
}
|
||||
if !slices.Equal(ended, []string{key + " failed"}) || !slices.Equal(turned, []string{key}) {
|
||||
t.Fatalf("said ended %v, turns freed %v: want %s's end said and its turn freed", ended, turned, key)
|
||||
}
|
||||
if _, held := u[UnfinishedKey("main", "app")]; !held {
|
||||
t.Fatalf("app is not left unfinished, so it rejoins no pipeline: %v", u)
|
||||
}
|
||||
}
|
||||
|
||||
// rawKept reads a memRaw as the reads do.
|
||||
type rawKept struct{ r *memRaw }
|
||||
|
||||
|
||||
@@ -347,6 +347,27 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A
|
||||
return nil, ended
|
||||
}
|
||||
|
||||
// keptGone ends p failed at its placing, "kept revision <r> gone", when the catalog no longer holds revision r a mover of
|
||||
// it is kept at (design 47 H2 H-A-496): each step not ended ends not asked, so its movers, whose steps never ran, are left
|
||||
// unfinished and rejoin the next pipeline (§786), and its end frees its turn under pipelines-at-once. It answers the steps
|
||||
// it ended, none when it had none; the pipeline's Ended says it ended.
|
||||
func keptGone(p *seat.Pipeline, r int64, err error, now time.Time) []int {
|
||||
why := fmt.Sprintf("kept revision %d gone", r)
|
||||
ended := []int{}
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
st.Outcome, st.Why, st.Ended, st.Ask, st.Shared = seat.StepResultOutcomeNotAsked, "the pipeline failed: "+why, &now, "", nil
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
}
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, fmt.Sprintf("%s (%v)", why, err), &now
|
||||
p.EndCount++
|
||||
return ended
|
||||
}
|
||||
|
||||
// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the
|
||||
// pipeline's end, through Ended; with none given it is said in the journal alone.
|
||||
func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error {
|
||||
@@ -416,18 +437,36 @@ func (s *Starter) create(key string, p seat.Pipeline) error {
|
||||
return s.Pipelines.Create(key, raw)
|
||||
}
|
||||
|
||||
// started is a pipeline's pipeline-started (§746): its key, branch, source, call, revision and movers.
|
||||
func started(p seat.Pipeline, at time.Time) seat.PipelineStarted {
|
||||
// started is a pipeline's pipeline-started (§746): its key, branch, source, call, revision and movers, and kept, every
|
||||
// catalog revision its movers are kept at (design 47 H2 H-A-496), so the indexer pins each while the pipeline runs.
|
||||
func started(p seat.Pipeline, at time.Time, kept []int64) seat.PipelineStarted {
|
||||
return seat.PipelineStarted{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call,
|
||||
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run}
|
||||
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run, Kept: kept}
|
||||
}
|
||||
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<revision> (§746).
|
||||
// keptList is every distinct revision of kept-revisions, in order (H-A-496).
|
||||
func keptList(kept map[string]int64) []int64 {
|
||||
var out []int64
|
||||
for _, r := range kept {
|
||||
if r > 0 && !slices.Contains(out, r) {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<revision> (§746), with the revisions its movers are kept at as
|
||||
// its record's kept-revisions holds them (H-A-496): a re-pin says it again with the full list (H-A-496 addendum).
|
||||
func (s *Starter) sayStartedPipeline(p seat.Pipeline, at time.Time) {
|
||||
if s.Say == nil {
|
||||
return
|
||||
}
|
||||
if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at)); err != nil {
|
||||
kept, err := keptOfStore(s.Store, p.Pipeline)
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its pipeline-started names no kept revision: %v", p.Pipeline, err)
|
||||
}
|
||||
if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at, keptList(kept))); err != nil {
|
||||
s.logf("pipeline-started of %s is not said: %v", p.Pipeline, err)
|
||||
}
|
||||
}
|
||||
@@ -445,6 +484,8 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its movers are placed as movers not kept: %v", p.Pipeline, err)
|
||||
}
|
||||
var gone int64
|
||||
var goneWhy error
|
||||
all := entries
|
||||
for i, m := range p.Movers {
|
||||
// A mover that left for a newer pipeline is placed there, never here (H-A-223, H-A-361).
|
||||
@@ -460,7 +501,13 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
entries := all
|
||||
if rev := kept[m.Module]; rev > 0 {
|
||||
var err error
|
||||
if entries, err = s.keptEntries(p.Branch, rev); err != nil {
|
||||
if entries, err = s.keptEntries(p.Branch, rev); errors.Is(err, ErrRevisionGone) {
|
||||
// A kept revision the catalog no longer holds is never read: the pipeline ends failed (H-A-496).
|
||||
if gone == 0 {
|
||||
gone, goneWhy = rev, err
|
||||
}
|
||||
continue
|
||||
} else if err != nil {
|
||||
// It waits for its kept revision to be read, said once and carried on the mover (design 47 H2 §787).
|
||||
if m.Unplaced != seat.MoverUnplacedKeptRevisionUnread || m.Revision != rev {
|
||||
s.logf("%s: %s waits for its kept revision %d to be read (%v)", p.Pipeline, m.Module, rev, err)
|
||||
@@ -520,6 +567,9 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
}
|
||||
}
|
||||
}
|
||||
if gone > 0 {
|
||||
return nil, unregistered, keptGone(p, gone, goneWhy, now), notAsked
|
||||
}
|
||||
if steps, ended := cycled(p, waits, unregistered, reg, all, now, s.Name); ended {
|
||||
return nil, unregistered, steps, notAsked
|
||||
} else {
|
||||
@@ -702,7 +752,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
}
|
||||
s.ran(key)
|
||||
s.sayNotAsked(p, notAsked)
|
||||
if len(ended) > 0 {
|
||||
if len(ended) > 0 || p.Ended != nil {
|
||||
return true, s.endedAtPlacing(p, ended)
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
@@ -878,16 +928,19 @@ func (s *Starter) placeWaiting(store Updater, due func(seat.Pipeline) bool, reg
|
||||
if err != nil {
|
||||
note(err)
|
||||
}
|
||||
if n == 0 {
|
||||
// A pipeline its placing ended (a kept revision gone, H-A-496) is said ended though it placed no step.
|
||||
if n == 0 && ended == nil {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
s.logf("%s: %d steps placed from %s (A-327)", key, n, from)
|
||||
// A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start,
|
||||
// said already, is said again harmlessly.
|
||||
s.sayStartedPipeline(p, now)
|
||||
if n > 0 {
|
||||
s.logf("%s: %d steps placed from %s (A-327)", key, n, from)
|
||||
// A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start,
|
||||
// said already, is said again harmlessly.
|
||||
s.sayStartedPipeline(p, now)
|
||||
}
|
||||
s.sayNotAsked(p, notAsked)
|
||||
if len(ended) > 0 {
|
||||
if ended != nil {
|
||||
if err := s.endedAtPlacing(p, ended); err != nil {
|
||||
note(err)
|
||||
}
|
||||
@@ -933,7 +986,7 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
|
||||
s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m)
|
||||
s.registrationMissing(m, key)
|
||||
}
|
||||
if len(p.Steps) == before {
|
||||
if len(p.Steps) == before && p.Ended == nil {
|
||||
// A mover that now waits for its kept revision, or no longer does, is written so (design 47 H2 §787).
|
||||
if unplacedMarks(p) == marks {
|
||||
return p, nil, nil, nil, 0, nil
|
||||
|
||||
Reference in New Issue
Block a user