From 070a5dd35b69f54a61474df25dbe1312b300226a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 10:34:12 +0100 Subject: [PATCH] pipeline runner: no pipeline is set aside for the runner's own new version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A take-in that changes the runner starts its pipelines as any take-in does; the state bounds' held keys, the wait's `on` value `runner version`, the finding pipeline-waits, the verb release (and the tools runner_release and runner_waits that only served it), the Held field and the runner's build-end and running hearing go. A pipeline a previous runner left set aside is placed and asked at the runner's start as a newly started one. Builds: design 47 G A-446 (replaces A-195, A-206, A-218, §266's hold) --- .../go/seats/meshpipelinerunner/contract.go | 33 +-- .../seats/mesh-pipeline-runner/seat.json | 56 +--- .../ts/seats/meshpipelinerunner/index.d.ts | 26 +- .../ts/seats/meshpipelinerunner/index.js | 1 - .../cmd/mesh-delivery-pipeline-tools/main.go | 29 -- .../mesh-delivery-pipeline-tools/main_test.go | 13 +- .../cmd/mesh-delivery-pipeline/buildends.go | 44 --- .../buildends_integration_test.go | 76 ----- .../mesh-delivery-pipeline/findingwriter.go | 56 ---- .../findingwriter_integration_test.go | 110 ------- .../findingwriter_test.go | 57 ---- .../cmd/mesh-delivery-pipeline/main.go | 32 +-- .../cmd/mesh-delivery-pipeline/steps.go | 57 ---- .../cmd/mesh-delivery-pipeline/verbs.go | 4 +- .../cmd/mesh-delivery-pipeline/verbs_test.go | 2 +- modules/mesh-delivery-pipeline/go.mod | 2 - .../internal/pipeline/ends.go | 46 --- .../internal/pipeline/hold.go | 269 ------------------ .../internal/pipeline/hold_test.go | 245 ---------------- .../internal/pipeline/read.go | 19 -- .../internal/pipeline/read_test.go | 16 -- .../internal/pipeline/start.go | 37 +-- .../internal/pipeline/start_test.go | 36 +++ .../mesh-delivery-pipeline/manifest_test.go | 6 +- modules/mesh-delivery-pipeline/module.json | 7 +- 25 files changed, 55 insertions(+), 1224 deletions(-) delete mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/buildends.go delete mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/buildends_integration_test.go delete mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter_integration_test.go delete mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter_test.go delete mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/ends.go delete mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/hold.go delete mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/hold_test.go diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index f9bdeca2..9eb79ad6 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -27,7 +27,6 @@ const ( VerbAccept = "accept" VerbCancel = "cancel" VerbAskAgain = "ask-again" - VerbRelease = "release" ) const ( @@ -46,7 +45,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain, VerbRelease} + return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain} } // Events is every event published on the seat's subject: a new copy each call. @@ -360,14 +359,6 @@ const ( GroupEndedOutcomeCancelled = "cancelled" ) -// Held is a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195). -type Held struct { - // the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195) - Fingerprint string `json:"fingerprint"` - // since when the pipeline is held, RFC 3339 - Since time.Time `json:"since"` -} - // KeptResult is a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8). type KeptResult struct { // sha256: of what the package's tests read @@ -452,7 +443,6 @@ type Pipeline struct { Fingerprint string `json:"fingerprint,omitempty"` // the gathered group, when the pipeline is its Group string `json:"group,omitempty"` - Held *Held `json:"held,omitempty"` Module string `json:"module"` Outcome string `json:"outcome"` // yes when the module is over its test budget (design 47 G section 419) @@ -731,7 +721,7 @@ type Wait struct { // for its end, how it ended: answered, ran, released, superseded How string `json:"how,omitempty"` Module string `json:"module"` - // what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206) + // what the step waits on (design 47 *Health and going back*) On string `json:"on"` // the pipeline's key, .., a branch escaped as a key's name is Pipeline string `json:"pipeline"` @@ -751,10 +741,9 @@ type Wait struct { // The values of Wait's on. const ( - WaitOnPerson = "person" - WaitOnBuildNode = "build node" - WaitOnRevisionView = "revision view" - WaitOnRunnerVersion = "runner version" + WaitOnPerson = "person" + WaitOnBuildNode = "build node" + WaitOnRevisionView = "revision view" ) // The values of Wait's step. @@ -991,18 +980,6 @@ type AskAgainAnswer struct { Result StepResult `json:"result"` } -// ReleaseInput is what release takes. -type ReleaseInput struct { - // why, written with the caller the runtime proves in the hand-act log - Why string `json:"why"` -} - -// ReleaseAnswer is what release answers. -type ReleaseAnswer struct { - // how many held pipelines were started - Released int64 `json:"released"` -} - // PipelineStartedEvent is the body of the event pipeline-started. type PipelineStartedEvent = PipelineStarted diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index 60c9f06c..4a178f8f 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -501,35 +501,6 @@ ] }, "class": "act" - }, - { - "name": "release", - "description": "The pipelines the runner holds for its new version started by hand with the running runner, whatever the new version's state; written in the hand-act log (design 47 G 276, A-206).", - "input": { - "type": "object", - "properties": { - "why": { - "type": "string", - "description": "why, written with the caller the runtime proves in the hand-act log" - } - }, - "required": [ - "why" - ] - }, - "answer": { - "type": "object", - "properties": { - "released": { - "type": "integer", - "description": "how many held pipelines were started" - } - }, - "required": [ - "released" - ] - }, - "class": "act" } ], "events": [ @@ -719,25 +690,6 @@ }, "description": "how many tests a step ran and carried, per level (ADR 0666 section 8)" }, - "Held": { - "type": "object", - "properties": { - "since": { - "type": "string", - "format": "date-time", - "description": "since when the pipeline is held, RFC 3339" - }, - "fingerprint": { - "type": "string", - "description": "the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195)" - } - }, - "required": [ - "since", - "fingerprint" - ], - "description": "a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195)" - }, "ToolchainUsed": { "type": "object", "properties": { @@ -1074,9 +1026,6 @@ "type": "boolean", "description": "yes when the module is over its test budget (design 47 G section 419)" }, - "held": { - "$ref": "#/defs/Held" - }, "widens": { "type": "array", "items": { @@ -1537,10 +1486,9 @@ "enum": [ "person", "build node", - "revision view", - "runner version" + "revision view" ], - "description": "what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206)" + "description": "what the step waits on (design 47 *Health and going back*)" }, "who": { "type": "string", diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index b878f1ab..f785f657 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -17,7 +17,6 @@ export declare const verbs: { readonly accept: "accept"; readonly cancel: "cancel"; readonly askAgain: "ask-again"; - readonly release: "release"; }; export declare const events: { @@ -154,14 +153,6 @@ export interface GroupEnded { "why"?: string; } -/** a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195) */ -export interface Held { - /** the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195) */ - "fingerprint": string; - /** since when the pipeline is held, RFC 3339 */ - "since": string; -} - /** a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8) */ export interface KeptResult { /** sha256: of what the package's tests read */ @@ -211,7 +202,6 @@ export interface Pipeline { "fingerprint"?: string; /** the gathered group, when the pipeline is its */ "group"?: string; - "held"?: Held; "module": string; "outcome": "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; /** yes when the module is over its test budget (design 47 G section 419) */ @@ -414,8 +404,8 @@ export interface Wait { /** for its end, how it ended: answered, ran, released, superseded */ "how"?: string; "module": string; - /** what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206) */ - "on": "person" | "build node" | "revision view" | "runner version"; + /** what the step waits on (design 47 *Health and going back*) */ + "on": "person" | "build node" | "revision view"; /** the pipeline's key, .., a branch escaped as a key's name is */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ @@ -606,18 +596,6 @@ export interface AskAgainAnswer { "result": StepResult; } -/** is what release takes. */ -export interface ReleaseInput { - /** why, written with the caller the runtime proves in the hand-act log */ - "why": string; -} - -/** is what release answers. */ -export interface ReleaseAnswer { - /** how many held pipelines were started */ - "released": number; -} - /** is the body of the event pipeline-started. */ export type PipelineStartedEvent = PipelineStarted; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js index 64a18312..e797aedb 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js @@ -17,7 +17,6 @@ export const verbs = { accept: "accept", cancel: "cancel", askAgain: "ask-again", - release: "release", }; export const events = { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index 6780b8ec..7177fb93 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -107,34 +107,6 @@ func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time) stdio. Run: func(map[string]any) (any, error) { return pipeline.Health(pipelines, consumers, now()), nil }} } -// WaitsTool is the runner's tool runner_waits: the pipelines held for its new version, with the fingerprint each awaits and since -// when (design 47 G §276, §539, A-195). -func WaitsTool(r pipeline.Reads) stdio.Tool { - return stdio.Tool{Name: Prefix + "_waits", - Description: "The pipelines the pipeline runner holds until its new version runs healthy: each with its pipeline, the " + - "fingerprint of the runner it awaits and since when.", - Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return r.Waits() }} -} - -// ReleaseTool is the runner's tool runner_release: it starts the pipelines held for the runner's new version with the running -// runner, through the seat's verb release, which the process that holds the pipelines answers and records in the hand-act log -// first (design 47 G §276, A-206). -func ReleaseTool(ask func(key string, body any) (json.RawMessage, error)) stdio.Tool { - return stdio.Tool{Name: Prefix + "_release", - Description: "Start now, with the running pipeline runner, the pipelines it holds until its new version runs healthy " + - "(runner_waits lists them), each wait ending released. Written in the hand-act log (mesh-hand-acts).", - Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{ - "why": map[string]any{"type": "string", "description": "why, in plain words"}}}, - Run: func(args map[string]any) (any, error) { - in, err := as[seat.ReleaseInput](args) - if err != nil { - return nil, err - } - return ask("seat:"+seat.Seat+"."+seat.VerbRelease, in) - }} -} - // StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states // pipelines and kept by `mesh/state.info`, step-outputs by the object store's fill and step-log by the log's; the runner's // process writes nothing for it. A store whose fill cannot be read is an error naming it. @@ -177,7 +149,6 @@ func main() { func(n string) (*stdio.LogInfo, error) { return stdio.Log(n).Info() }, time.Now) r := pipeline.Reads{Pipelines: kept{stdio.State(seat.StatePipelines)}, Kept: kept{stdio.State(seat.StateKept)}, Fill: fill} tools := append(Tools(r), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) - tools = append(tools, WaitsTool(r), ReleaseTool(stdio.Ask)) tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask}, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...) tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now)) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index 1e4e074c..a70454ad 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" - "fmt" "git.novox.be/novox/mesh-sdk/go/buscalls" "strings" "testing" @@ -34,21 +33,11 @@ func TestTools(t *testing.T) { } // RN25 (§468, §76; A-164): the health check is the bundle's runner_health. names = append(names, HealthTool(none{}, none{}, nil).Name) - // A-206: the tool runner_release asks the seat's verb release with its why. - var asked string - rel := ReleaseTool(func(key string, body any) (json.RawMessage, error) { - asked = key + fmt.Sprintf(" %+v", body) - return json.RawMessage(`{"released":1}`), nil - }) - if _, err := rel.Run(map[string]any{"why": "w"}); err != nil || asked != "seat:mesh-pipeline-runner.release {Why:w}" { - t.Errorf("runner_release asked %q, %v", asked, err) - } - names = append(names, rel.Name) // §606, §549: the unlogged entries are listed and appended by hand. for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) { names = append(names, tl.Name) } - want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_health runner_release runner_unlogged runner_unlogged_append" + want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_health runner_unlogged runner_unlogged_append" if got := strings.Join(names, " "); got != want { t.Errorf("tools %q, want %q", got, want) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/buildends.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/buildends.go deleted file mode 100644 index fbb5149c..00000000 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/buildends.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "context" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/process" - - "mesh-delivery-pipeline/internal/pipeline" -) - -// hearBuildEnds hears the builder's stream of build ends, kept last per subject and read whole at the start, then followed -// (design 47 G A-181, §556): when the build of the runner's new version ended is the time the wait's bound counts from, and -// each end heard for a runner held pipelines await starts their clock (§266, A-206). A stream it cannot hear is said, and the -// bound of such a wait is not armed until the runner starts again. -func hearBuildEnds(ctx context.Context, cred process.Account, js jetstream.JetStream, ends *pipeline.BuildEnds, st *pipeline.Starter, store pipeline.Updater, - logf func(string, ...any)) { - stream := catalogue.StreamBusName(builds.BuildsStream) - cc, err := cred.HearStream(ctx, js, stream, func(m jetstream.Msg) { - defer m.Ack() - md, err := m.Metadata() - if err != nil { - logf("%s has no time the stream stamped: %v", m.Subject(), err) - return - } - module, fp, ok := ends.Heard(m.Data(), md.Timestamp) - if !ok { - return - } - if n, err := st.Clocked(store, module, fp); err != nil { - logf("the clock of the %d pipelines held for %s: %v", n, fp, err) - } else if n > 0 { - logf("the build of %s ended: the bound of the %d pipelines held for it counts from it", fp, n) - } - }, logf) - if err != nil { - logf("the stream %s is not heard, so no wait for the runner's new version has its bound armed until the runner starts again: %v", stream, err) - return - } - go func() { <-ctx.Done(); cc.Stop() }() -} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/buildends_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/buildends_integration_test.go deleted file mode 100644 index cb17252d..00000000 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/buildends_integration_test.go +++ /dev/null @@ -1,76 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" - seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/process" - "git.novox.be/novox/mesh-sdk/go/testbus" - - "mesh-delivery-pipeline/internal/pipeline" -) - -// The runner hears the builder's stream of build ends, kept last per subject, read whole at its start: the end of its new version's -// build, published before it started, starts the clock of the pipelines held for it, counted from the time the stream stamped the end -// (design 47 G A-181, §266, A-206). -func TestTheRunnerHearsTheBuildEndOfItsNewVersionAtItsStart(t *testing.T) { - nc, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer nc.Close() - js, err := jetstream.New(nc) - if err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - stream, subject := catalogue.StreamBusName(builds.BuildsStream), process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>") - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil { - t.Fatal(err) - } - published := time.Now().UTC() - if _, err := js.Publish(ctx, process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEventName("mesh-delivery-pipeline", "src2:new")), - []byte(`{"module":"mesh-delivery-pipeline","fingerprint":"src2:new","outcome":"done"}`)); err != nil { - t.Fatal(err) - } - cred := process.Account{Node: "n1", Module: "mesh-delivery-pipeline"} - durable := cred.SeatEventConsumerConfig([]string{subject}, true) - durable.DeliverPolicy = jetstream.DeliverNewPolicy - if _, err := js.CreateOrUpdateConsumer(ctx, stream, durable); err != nil { - t.Fatal(err) - } - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "pipelines", History: 1}) - if err != nil { - t.Fatal(err) - } - store := pipelineStore{ctx: ctx, kv: kv} - held, _ := json.Marshal(seat.Pipeline{Pipeline: "main.4.app", Branch: "main", Module: "app", Outcome: seat.PipelineOutcomeRunning, - Held: &seat.Held{Since: published, Fingerprint: "src2:new"}}) - if _, err := kv.Create(ctx, "main.4.app", held); err != nil { - t.Fatal(err) - } - ends := &pipeline.BuildEnds{} - armed := make(chan time.Time, 1) - st := &pipeline.Starter{Self: "mesh-delivery-pipeline", Ends: ends.At, Say: func(string, any) error { return nil }, - Hold: func(fp string, from time.Time) error { armed <- from; return nil }} - hearBuildEnds(ctx, cred, js, ends, st, store, t.Logf) - select { - case from := <-armed: - if from.Before(published.Add(-time.Minute)) || from.After(time.Now().Add(time.Second)) { - t.Errorf("the bound counts from %s, not from the end published at %s", from, published) - } - case <-ctx.Done(): - t.Fatal("the end of the new runner's build was not heard at the start") - } -} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter.go index adce7906..ca5860a3 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter.go @@ -2,9 +2,6 @@ package main import ( "context" - "fmt" - "slices" - "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -12,34 +9,11 @@ import ( condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" - - seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) // findingsWriter is the runner's finding writer as the mesh-sdk makes it. type findingsWriter = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent] -// KindPipelineWaits is the finding of pipelines held for the runner's new version that the wait's bound started with the running -// runner (design 47 G §266, A-206). -const KindPipelineWaits = "pipeline-waits" - -// awaitedSaid opens the evidence entry of a pipeline-waits finding that names the awaited runner fingerprint, `awaited ` -// (A-276): the runner clears a finding by that entry alone, never by its summary. -const awaitedSaid = "awaited " - -// lateFinding is the finding pipeline-waits of one pipeline the bound's expiry started: scope module, id its key -// `..`, naming the fingerprint of the new runner it waited for and since when, the fingerprint also in one -// evidence entry `awaited ` (A-276). -func lateFinding(p seat.Pipeline, held seat.Held, runner, node string, at time.Time) condseat.Finding { - return condseat.Finding{Scope: views.FindingScope, ID: p.Pipeline, Kind: KindPipelineWaits, Severity: condseat.FindingSeverityWarning, - Evidence: []condseat.Evidence{{At: at, Said: awaitedSaid + held.Fingerprint}}, - Headline: "the new pipeline runner did not run healthy in time", - Summary: fmt.Sprintf("the pipeline %s was held since %s for the runner at %s to run healthy; the bound passed and the running runner runs it", - p.Pipeline, held.Since.UTC().Format(time.RFC3339), held.Fingerprint), - Source: "bound of the wait for the new runner", Observer: runner, At: at, - Subject: condseat.Subject{Scope: views.FindingScope, ID: p.Module, Machine: node}} -} - // findingWriter is the runner's mesh-sdk held finding writer (design 47 G §461, §689, A-29): the runner raises its findings // as they happen (a failed head, §33), so the writer holds what it has written and its statement says exactly that, with the // reader's open findings. The writer @@ -58,33 +32,3 @@ func findingWriter(ctx context.Context, cred process.Account, conn *nats.Conn, j } return w, nil } - -// heldFindings is what clearAwaited needs of the runner's held finding writer: what it holds and a clear. -type heldFindings interface { - Held() []condseat.Finding - Clear(ctx context.Context, scope, id, kind, why string) error -} - -// clearAwaited clears exactly the pipeline-waits findings whose evidence names fp awaited (A-218, A-276): the node-engine seat's -// running says the runner at fp runs healthy. The writer holds what its state findings hold from its start, so a finding raised -// before a restart is cleared too. It says how many it cleared, and the first clear that failed. -func clearAwaited(ctx context.Context, w heldFindings, fp string) (int, error) { - if w == nil || fp == "" { - return 0, nil - } - n := 0 - var first error - for _, f := range w.Held() { - if f.Kind != KindPipelineWaits || !slices.ContainsFunc(f.Evidence, func(e condseat.Evidence) bool { return e.Said == awaitedSaid+fp }) { - continue - } - if err := w.Clear(ctx, f.Scope, f.ID, f.Kind, "the runner at "+fp+" runs healthy"); err != nil { - if first == nil { - first = err - } - continue - } - n++ - } - return n, first -} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter_integration_test.go deleted file mode 100644 index d4936c40..00000000 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter_integration_test.go +++ /dev/null @@ -1,110 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" - "git.novox.be/novox/mesh-sdk/go/process" - "git.novox.be/novox/mesh-sdk/go/testbus" - - "mesh-delivery-pipeline/internal/pipeline" -) - -// Design 47 G A-218, A-276: a pipeline-waits finding raised before the runner restarted is held again by the new process's writer, -// seeded from the state findings, and is cleared when the node-engine seat's running says the awaited runner fingerprint healthy; -// the finding awaiting another fingerprint stands. -func TestAPipelineWaitsFindingIsClearedAfterARestartWhenItsAwaitedRunnerRunsHealthy(t *testing.T) { - url := testbus.URL(t) - nc, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer nc.Close() - js, err := jetstream.New(nc) - if err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - const module = "mesh-delivery-pipeline" - states := map[string]jetstream.KeyValue{} - for _, state := range []string{"findings", "pipelines", "bounds"} { - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}) - if err != nil { - t.Fatal(err) - } - states[state] = kv - } - // Where the writer says its findings, as the bus keeps the module's events. - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.>"}}); err != nil { - t.Fatal(err) - } - cred := process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"} - conn, err := process.Connect(cred) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - since := time.Now().UTC().Add(-time.Hour) - before, err := findingWriter(ctx, cred, nc, js, conn.State("findings"), t.Logf) - if err != nil { - t.Fatal(err) - } - for _, p := range []struct{ key, module, fp string }{{"main.4.app", "app", "src2:new"}, {"main.5.web", "web", "src2:newer"}} { - if err := before.Write(ctx, lateFinding(seat.Pipeline{Pipeline: p.key, Module: p.module, Branch: "main"}, - seat.Held{Since: since, Fingerprint: p.fp}, module, "n1", since.Add(30*time.Minute))); err != nil { - t.Fatal(err) - } - } - before.Close() - keys := func() map[string]bool { - out := map[string]bool{} - ks, err := states["findings"].Keys(ctx) - if err != nil && err != jetstream.ErrNoKeysFound { - t.Fatal(err) - } - for _, k := range ks { - out[k] = true - } - return out - } - app, web := "module.main.4.app.pipeline-waits", "module.main.5.web.pipeline-waits" - if k := keys(); !k[app] || !k[web] { - t.Fatalf("the state findings holds %v before the restart, want %s and %s", k, app, web) - } - - // The restart: a new process's writer holds what its state findings hold. - after, err := findingWriter(ctx, cred, nc, js, conn.State("findings"), t.Logf) - if err != nil { - t.Fatal(err) - } - defer after.Close() - if len(after.Held()) != 2 { - t.Fatalf("the writer after the restart holds %+v, want both findings", after.Held()) - } - hub := &oneHub{} - starter := &pipeline.Starter{Logf: t.Logf} - if err := hearRunning(ctx, hub, module, starter, pipelineStore{ctx: ctx, kv: states["pipelines"]}, states["bounds"], after, - func(string) (pipeline.Half, bool) { return pipeline.Half{}, false }, t.Logf); err != nil { - t.Fatal(err) - } - healthy := true - raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: "n1", Module: module, Fingerprint: "src2:new", Healthy: &healthy, At: time.Now().UTC()}) - var acks []string - hub.handle(dataMsg{data: raw, acks: &acks}) - if k := keys(); k[app] || !k[web] { - t.Errorf("after the runner at src2:new runs healthy the state findings holds %v: want %s cleared and %s standing", k, app, web) - } - if held := after.Held(); len(held) != 1 || held[0].ID != "main.5.web" { - t.Errorf("the writer holds %+v, want only the finding awaiting src2:newer", held) - } -} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter_test.go deleted file mode 100644 index b6ec439b..00000000 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "context" - "slices" - "strings" - "testing" - "time" - - condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" - "git.novox.be/novox/mesh-sdk/go/views" - - seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" -) - -// The bound's expiry's finding is pipeline-waits, scope module, id the pipeline's key, naming the awaited fingerprint and since -// when (design 47 G §266, A-206); it passes the writer's own check of a finding. -func TestLateFindingIsPipelineWaitsOfTheModuleScope(t *testing.T) { - since := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) - f := lateFinding(seat.Pipeline{Pipeline: "main.4.app", Module: "app", Branch: "main"}, seat.Held{Since: since, Fingerprint: "src2:new"}, - "mesh-delivery-pipeline", "n1", since.Add(30*time.Minute)) - if f.Kind != "pipeline-waits" || f.Scope != views.FindingScope || f.ID != "main.4.app" || f.Severity != condseat.FindingSeverityWarning { - t.Errorf("finding %+v", f) - } - if !strings.Contains(f.Summary, "src2:new") || !strings.Contains(f.Summary, "2026-11-11T12:00:00Z") { - t.Errorf("summary %q", f.Summary) - } - if err := views.CheckFinding(f); err != nil { - t.Error(err) - } - // A-276: the awaited fingerprint is in one evidence entry, `awaited `. - if len(f.Evidence) != 1 || f.Evidence[0].Said != "awaited src2:new" { - t.Errorf("evidence %+v, want one entry `awaited src2:new`", f.Evidence) - } - // A-218, A-276: a healthy running of the runner at a fingerprint clears exactly the pipeline-waits findings whose evidence - // names it awaited: not one awaiting another, not another kind naming it, never by the summary. - other := lateFinding(seat.Pipeline{Pipeline: "main.5.web", Module: "web"}, seat.Held{Since: since, Fingerprint: "src2:newer"}, "r", "n1", since) - other.Summary = f.Summary // a summary naming src2:new is never read - kind := f - kind.Kind, kind.ID = "step-failed", "main.4.app.build" - held := &memFindings{held: []condseat.Finding{f, other, kind}} - if n, err := clearAwaited(context.Background(), held, "src2:new"); err != nil || n != 1 || !slices.Equal(held.cleared, []string{"module.main.4.app.pipeline-waits"}) { - t.Errorf("cleared %d (%v): %v, want the one pipeline-waits awaiting src2:new", n, err, held.cleared) - } -} - -// memFindings is a held finding writer's Held and Clear. -type memFindings struct { - held []condseat.Finding - cleared []string -} - -func (m *memFindings) Held() []condseat.Finding { return m.held } -func (m *memFindings) Clear(_ context.Context, scope, id, kind, _ string) error { - m.cleared = append(m.cleared, scope+"."+id+"."+kind) - return nil -} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9f442ba7..0a836eb8 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -137,20 +137,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf} // Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked // for its step. - ends := &pipeline.BuildEnds{} starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say, Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module, Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), - // The runner's own module and fingerprint (A-149): a take-in giving it another holds the pipelines it starts (§65, A-195). - Self: cred.Module, SelfFp: getenv("MESH_FINGERPRINT"), Awaiting: pipeline.Awaiting(pipelineStore{ctx: ctx, kv: pipelines}), - Ends: ends.At, Hold: armHeld(ctx, bounds), - // The bound's expiry writes the finding pipeline-waits for each pipeline it started (§266, A-206). - Late: func(p seat.Pipeline, held seat.Held) { - module, node := rv.Findings.ModuleNode() - if err := rv.Findings.Write(ctx, lateFinding(p, held, module, node, time.Now().UTC())); err != nil { - logger.Printf("the finding pipeline-waits of %s is not written: %v", p.Pipeline, err) - } - }, Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has} placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error { n, err := starter.PlaceRun(ev, half, rv.Catalog.View) @@ -162,10 +151,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if !ok { return } - // Pipelines held for the fingerprint this runner runs are started now: it is the new runner (§19, A-195). - if n, err := starter.Release(pipelineStore{ctx: ctx, kv: pipelines}, starter.SelfFp, "ran", rv.Catalog.View); err != nil { - logger.Printf("the pipelines held for the runner at %s: %d released, then %v", starter.SelfFp, n, err) - } // A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327). if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err) @@ -234,23 +219,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil { return err } - store := pipelineStore{ctx: ctx, kv: pipelines} expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) - if err := watchBounds(ctx, bounds, func(key string) { - if !pipeline.IsHeldKey(key) { - expired(key) - return - } - if n, err := starter.ReleaseExpired(store, key, rv.Catalog.View); err != nil { - logger.Printf("the pipelines held for the runner, whose %s passed: %d released, then %v", key, n, err) - } else { - logger.Printf("the runner's new version did not run healthy within %s: %d held pipelines started with the running runner", pipeline.HeldBound, n) - } - }, logger.Printf); err != nil { - return err - } - hearBuildEnds(ctx, cred, js, ends, starter, store, logger.Printf) - if err := hearRunning(ctx, heard, cred.Module, starter, store, bounds, rv.Findings, rv.Catalog.View, logger.Printf); err != nil { + if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil { return err } if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index 813e01a5..ee7d7ded 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -6,11 +6,9 @@ import ( "errors" "fmt" "strings" - "time" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" @@ -178,61 +176,6 @@ func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm { } } -// armHeld is the bound of the pipelines held for the runner's new version: a key of the state bounds named for the awaited -// fingerprint, with its own time to live of 30 minutes counted from the end of that runner's build (design 47 G §266, A-195, A-77, -// A-206). -func armHeld(ctx context.Context, bounds jetstream.KeyValue) func(string, time.Time) error { - return func(fp string, from time.Time) error { - _, err := bounds.Create(ctx, pipeline.HeldKey(fp), []byte(`{"fingerprint":"`+fp+`"}`), jetstream.KeyTTL(pipeline.HeldTTL(from, time.Now()))) - if errors.Is(err, jetstream.ErrKeyExists) { - return nil - } - return err - } -} - -// hearRunning releases the pipelines held for the runner's new version when the node-engine seat says the runner's own module -// runs that fingerprint healthy, on any node (§266, §276, A-195); its bound is then deleted, and the pipeline-waits findings -// that await it are cleared (A-218, A-276). A running with no health reads as healthy (§520). Heard again when the release or -// a clear could not be written. -func hearRunning(ctx context.Context, hub handlers, self string, st *pipeline.Starter, store pipeline.Updater, bounds jetstream.KeyValue, - findings heldFindings, view func(string) (pipeline.Half, bool), logf func(string, ...any)) error { - return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) { - var ev nodeengine.Running - if err := json.Unmarshal(m.Data(), &ev); err != nil { - logf("%s is no running: %v", m.Subject(), err) - _ = m.Term() - return - } - if ev.Module != self || ev.Stopped || ev.Fingerprint == "" || (ev.Healthy != nil && !*ev.Healthy) { - _ = m.Ack() - return - } - n, err := st.Release(store, ev.Fingerprint, "ran", view) - if err != nil { - logf("the pipelines held for %s: %d released, then %v; heard again", ev.Fingerprint, n, err) - _ = m.Nak() - return - } - if n > 0 { - logf("the runner runs %s on %s: %d held pipelines released", ev.Fingerprint, ev.Node, n) - if err := bounds.Delete(ctx, pipeline.HeldKey(ev.Fingerprint)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - logf("the bound of the wait for %s is not deleted: %v", ev.Fingerprint, err) - } - } - cleared, err := clearAwaited(ctx, findings, ev.Fingerprint) - if cleared > 0 { - logf("the runner runs %s on %s: %d pipeline-waits findings cleared", ev.Fingerprint, ev.Node, cleared) - } - if err != nil { - logf("a pipeline-waits finding awaiting %s is not cleared: %v; heard again", ev.Fingerprint, err) - _ = m.Nak() - return - } - _ = m.Ack() - }) -} - // watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32, // §535, A-77), and says a drop no holder answered unanswered (A-222): no timer in the process. func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go index 4c62a4a7..27105fb6 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go @@ -41,8 +41,7 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v)) } -// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, ask-again and -// release (§64, §306; RN12, RN14; A-206). +// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again. func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { return map[string]process.SeatHandler{ seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }), @@ -52,7 +51,6 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }), seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }), seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }), - seat.VerbRelease: verbOf(func(c string, in seat.ReleaseInput) (any, error) { return v.Release(c, in) }), } } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go index d71a8898..9e2d5d28 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go @@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) { got = append(got, v) } slices.Sort(got) - want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbRelease, seat.VerbReview, seat.VerbStart, seat.VerbRun} + want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun} slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("the process answers %v, want %v", got, want) diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index a55bcf38..50349fa3 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -9,13 +9,11 @@ require ( ) require ( - git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/ends.go b/modules/mesh-delivery-pipeline/internal/pipeline/ends.go deleted file mode 100644 index cb7ae9e1..00000000 --- a/modules/mesh-delivery-pipeline/internal/pipeline/ends.go +++ /dev/null @@ -1,46 +0,0 @@ -package pipeline - -import ( - "encoding/json" - "sync" - "time" -) - -// BuildEnds are the done builds' ends heard from the builder's stream of build ends (event build.ended, kept last per subject -// and read whole at the start, design 47 G A-181): when each build, by module and fingerprint, ended, which is the time a held -// wait's bound counts from (§266, A-206). -type BuildEnds struct { - mu sync.Mutex - at map[string]time.Time -} - -// buildEnd is the part of the body of build.ended the runner reads. -type buildEnd struct { - Module string `json:"module"` - Fingerprint string `json:"fingerprint"` - Outcome string `json:"outcome"` -} - -// Heard keeps the end a message of the stream says, at the time the stream stamped it; a failed end, or a body naming no module -// and fingerprint, is kept for nothing. It says the module and fingerprint it kept. -func (e *BuildEnds) Heard(data []byte, at time.Time) (module, fp string, ok bool) { - var b buildEnd - if err := json.Unmarshal(data, &b); err != nil || b.Module == "" || b.Fingerprint == "" || b.Outcome == "failed" { - return "", "", false - } - e.mu.Lock() - defer e.mu.Unlock() - if e.at == nil { - e.at = map[string]time.Time{} - } - e.at[b.Module+"\x00"+b.Fingerprint] = at.UTC() - return b.Module, b.Fingerprint, true -} - -// At is when the build of the module at the fingerprint ended, false when no end was heard. -func (e *BuildEnds) At(module, fp string) (time.Time, bool) { - e.mu.Lock() - defer e.mu.Unlock() - t, ok := e.at[module+"\x00"+fp] - return t, ok -} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/hold.go b/modules/mesh-delivery-pipeline/internal/pipeline/hold.go deleted file mode 100644 index 1b01137e..00000000 --- a/modules/mesh-delivery-pipeline/internal/pipeline/hold.go +++ /dev/null @@ -1,269 +0,0 @@ -package pipeline - -import ( - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "slices" - "time" - - runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" - seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" -) - -// HeldBound is how long the pipelines held for the runner's new version wait for it to run healthy (design 47 G §266: 30 -// minutes for the runner's `pipeline` step, A-195): a key of the state `bounds` with its own time to live, never a timer. -const HeldBound = 30 * time.Minute - -// HeldPrefix starts the key of a held wait's bound. -const HeldPrefix = "held-" - -// HeldKey is the bound's key for the awaited fingerprint: a fingerprint holds characters a key's name does not. -func HeldKey(fp string) string { - sum := sha256.Sum256([]byte(fp)) - return HeldPrefix + hex.EncodeToString(sum[:12]) -} - -// IsHeldKey says whether a key of the state `bounds` is a held wait's bound. -func IsHeldKey(key string) bool { - return len(key) > len(HeldPrefix) && key[:len(HeldPrefix)] == HeldPrefix -} - -// Awaiting is the awaited fingerprint of the pipelines held in store, "" and false when none is held (design 47 G §266, A-195): -// a pipeline started while one is held is held for the same version, so it never runs on the old runner. -func Awaiting(store Updater) func() (string, bool) { - return func() (string, bool) { - pkeys, err := store.Keys("*.*.*") - if err != nil { - return "", false - } - for _, key := range pkeys { - if p, _, err := store.Get(key); err == nil && p.Held != nil && p.Outcome == seat.PipelineOutcomeRunning { - return p.Held.Fingerprint, true - } - } - return "", false - } -} - -// changesSelf says whether a take-in's entry is the runner's own module at a fingerprint other than the one that runs: its -// pipelines are held for the new runner (design 47 G §65, §266, A-195). -func (s *Starter) changesSelf(module, fp string) bool { - return s.Self != "" && module == s.Self && fp != "" && fp != s.SelfFp -} - -// startHeld writes b's pipeline held for the runner's version awaiting, its steps neither placed nor asked, and says -// wait-held with step `pipeline`; pipeline-started is said when it is released (§74, §266). -func (s *Starter) startHeld(b Begin, key string, now time.Time, awaiting string) (bool, error) { - p := newPipeline(b, key, now) - p.Held = &seat.Held{Since: now, Fingerprint: awaiting} - if err := s.create(key, p); errors.Is(err, ErrExists) { - s.logf("%s is started already: this start writes nothing", key) - return false, nil - } else if err != nil { - return false, fmt.Errorf("the pipeline %s is not written: %w", key, err) - } - from := s.clockOf(awaiting) - if from != nil && s.Hold != nil { - if err := s.Hold(awaiting, *from); err != nil { - s.logf("the bound of the wait for the runner at %s is not armed: %v", awaiting, err) - } - } - w := s.waitOf(p, now, "") - w.ClockFrom = from - if err := s.Say(runnerclient.Subject(seat.EventWaitHeld, b.Module, b.Branch, commitOrGroup(p)), w); err != nil { - s.logf("wait-held of %s is not said: %v", key, err) - } - return true, nil -} - -// clockOf is when the build of the runner at fp ended, the time the bound counts from, nil while it has not (§266, A-181, A-206). -func (s *Starter) clockOf(fp string) *time.Time { - if s.Ends == nil { - return nil - } - if at, ok := s.Ends(s.Self, fp); ok { - return &at - } - return nil -} - -// Clocked starts the clock of the pipelines held for the runner at fp, the end of whose build was heard: it arms the bound from that -// end and says wait-held again for each with `clock-from`, so the watchman times it by the bound (§269, A-206). It says how many -// pipelines it said, and nothing when none is held for fp or the end is not known. -func (s *Starter) Clocked(store Updater, module, fp string) (int, error) { - if module != s.Self { - return 0, nil - } - from := s.clockOf(fp) - if from == nil { - return 0, nil - } - pkeys, err := store.Keys("*.*.*") - if err != nil { - return 0, fmt.Errorf("the pipelines are not listed: %w", err) - } - said := 0 - var first error - for _, key := range pkeys { - p, _, err := store.Get(key) - if err != nil || p.Held == nil || p.Outcome != seat.PipelineOutcomeRunning || p.Held.Fingerprint != fp { - continue - } - if said == 0 && s.Hold != nil { - if err := s.Hold(fp, *from); err != nil { - first = fmt.Errorf("the bound of the wait for the runner at %s is not armed: %w", fp, err) - } - } - w := s.waitOf(p, p.Held.Since, "") - w.ClockFrom = from - if err := s.Say(runnerclient.Subject(seat.EventWaitHeld, p.Module, p.Branch, commitOrGroup(p)), w); err != nil && first == nil { - first = fmt.Errorf("wait-held of %s is not said: %w", key, err) - } - said++ - } - return said, first -} - -func commitOrGroup(p seat.Pipeline) string { - if p.Group != "" { - return p.Group - } - return p.Commit -} - -func (s *Starter) waitOf(p seat.Pipeline, since time.Time, how string) seat.Wait { - return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint, Group: p.Group, - Revision: p.Revision, Source: p.Source, Step: seat.WaitStepPipeline, On: seat.WaitOnRunnerVersion, Since: since, How: how} -} - -// Release starts the pipelines held for fingerprint fp ("" for every held one) as the new runner running healthy lets -// them, or the bound's expiry or a hand release (how: `ran` or `released`; §266, §276): each has its steps placed and asked and -// says wait-ended and pipeline-started. A pipeline whose branch has no view yet stays held, and the first error is returned -// so the event is heard again. It says how many it started. -func (s *Starter) Release(store Updater, fp, how string, view func(branch string) (Half, bool)) (int, error) { - pkeys, err := store.Keys("*.*.*") - if err != nil { - return 0, fmt.Errorf("the pipelines are not listed: %w", err) - } - now := time.Now().UTC() - if s.Now != nil { - now = s.Now().UTC() - } - started := 0 - var first error - note := func(err error) { - if first == nil { - first = err - } - } - for _, key := range pkeys { - p, rev, err := store.Get(key) - if err != nil || p.Held == nil || p.Outcome != seat.PipelineOutcomeRunning || (fp != "" && p.Held.Fingerprint != fp) { - continue - } - half, ok := view(p.Branch) - if !ok { - note(fmt.Errorf("the catalog half of %s is not held yet: %s stays held", p.Branch, key)) - continue - } - held := *p.Held - p.Held = nil - var asks []Asked - if s.Registration != nil { - if reg, ok := s.Registration(p.Module); ok { - p.Steps, asks = Place(p, reg, half.Entries, now, s.Name) - } - } - if _, err := store.Update(key, p, rev); err != nil { - note(fmt.Errorf("the held pipeline %s is not released: %w", key, err)) - continue - } - started++ - if err := s.Say(runnerclient.Subject(seat.EventWaitEnded, p.Module, p.Branch, commitOrGroup(p)), s.waitOf(p, held.Since, how)); err != nil { - s.logf("wait-ended of %s is not said: %v", key, err) - } - ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: p.Branch, Source: p.Source, Module: p.Module, Commit: p.Commit, - Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision} - if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, commitOrGroup(p)), ev); err != nil { - s.logf("pipeline-started of %s is not said: %v", key, err) - } - if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil { - note(err) - } - } - } - return started, first -} - -// ReleaseExpired starts the pipelines held for the fingerprint whose bound key expired (§266, A-195): the 30 minutes passed -// without the new runner running healthy, so the running runner runs them, and each started is said late through Late (A-206). -func (s *Starter) ReleaseExpired(store Updater, key string, view func(branch string) (Half, bool)) (int, error) { - pkeys, err := store.Keys("*.*.*") - if err != nil { - return 0, fmt.Errorf("the pipelines are not listed: %w", err) - } - fp := "" - held := map[string]seat.Held{} - for _, k := range pkeys { - if p, _, err := store.Get(k); err == nil && p.Held != nil && p.Outcome == seat.PipelineOutcomeRunning && HeldKey(p.Held.Fingerprint) == key { - fp, held[k] = p.Held.Fingerprint, *p.Held - } - } - if fp == "" { - return 0, nil - } - n, err := s.Release(store, fp, "released", view) - for k, h := range held { - if p, _, gerr := store.Get(k); gerr == nil && p.Held == nil && s.Late != nil { - s.Late(p, h) - } - } - return n, err -} - -// heldFingerprints are the fingerprints of the runner the held pipelines in store await, each once. -func heldFingerprints(store Updater) []string { - pkeys, err := store.Keys("*.*.*") - if err != nil { - return nil - } - var fps []string - for _, key := range pkeys { - if p, _, err := store.Get(key); err == nil && p.Held != nil && p.Outcome == seat.PipelineOutcomeRunning && !slices.Contains(fps, p.Held.Fingerprint) { - fps = append(fps, p.Held.Fingerprint) - } - } - return fps -} - -// Release starts by hand the pipelines held for the runner's new version with the running runner, recorded in the hand-act -// log first (§276, A-206): each says wait-ended `released` and pipeline-started, and the bound of each wait is dropped. It -// says how many it started. -func (v Verbs) Release(_ string, in seat.ReleaseInput) (seat.ReleaseAnswer, error) { - if v.Starter == nil || v.View == nil { - return seat.ReleaseAnswer{}, errors.New("the runner cannot release a pipeline: it holds no starter or no catalog half") - } - if err := v.record(seat.VerbRelease, "started the pipelines held for the new runner by hand", in.Why); err != nil { - return seat.ReleaseAnswer{}, err - } - fps := heldFingerprints(v.Hearer.Store) - n, err := v.Starter.Release(v.Hearer.Store, "", "released", v.View) - if n > 0 && v.Drop != nil { - for _, fp := range fps { - if derr := v.Drop(HeldKey(fp)); derr != nil && err == nil { - err = fmt.Errorf("the bound of the wait for %s is not dropped: %w", fp, derr) - } - } - } - return seat.ReleaseAnswer{Released: int64(n)}, err -} - -// HeldTTL is how long the bound of a wait that counts from the build end at from still has to run at now: the 30 minutes less what passed since -// the end, at least a second, the least a key's time to live holds (design 47 G §266, A-206). -func HeldTTL(from, now time.Time) time.Duration { - return max(HeldBound-now.Sub(from), time.Second) -} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/hold_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/hold_test.go deleted file mode 100644 index 39149241..00000000 --- a/modules/mesh-delivery-pipeline/internal/pipeline/hold_test.go +++ /dev/null @@ -1,245 +0,0 @@ -package pipeline - -import ( - "encoding/json" - "strings" - "testing" - "time" - - catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" -) - -// memCreated is memStore with the Create a start writes through. -type memCreated struct{ memStore } - -func (m memCreated) Create(key string, value []byte) error { - if _, ok := m.memStore[key]; ok { - return ErrExists - } - var p seat.Pipeline - if err := json.Unmarshal(value, &p); err != nil { - return err - } - m.memStore[key] = struct { - p seat.Pipeline - rev uint64 - }{p, 1} - return nil -} - -// A take-in giving the runner's own module a new fingerprint holds every pipeline it starts, a later take-in's too while one is -// held, with no step asked; the new runner running healthy releases them, placing and asking their steps (design 47 G §65, §266, -// §276, §534; A-195). -func TestPipelinesAreHeldForTheNewRunnerAndReleasedWhenItRuns(t *testing.T) { - reg, entries := placeFixture() - entries["runner"] = catalog.CatalogEntry{Module: "runner"} - store := memCreated{memStore{}} - asks := &stepAsks{} - var said []string - var bounds []string - s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Self: "runner", SelfFp: "src2:old", - Awaiting: Awaiting(store.memStore), Ends: func(string, string) (time.Time, bool) { return time.Unix(9, 0), true }, - Hold: func(fp string, _ time.Time) error { bounds = append(bounds, fp); return nil }, - Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }, - Say: func(subject string, _ any) error { - said = append(said, subject[strings.Index(subject, ".event.")+7:]) - return nil - }} - view := func(string) (Half, bool) { return Half{Entries: entries}, true } - n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{ - {Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "runner", Commit: "c2", Fingerprint: "src2:new"}}}, view) - if err != nil || n != 2 { - t.Fatalf("take-in = %d, %v", n, err) - } - p, _, _ := store.Get("main.4.app") - if p.Held == nil || p.Held.Fingerprint != "src2:new" || len(p.Steps) != 0 || len(asks.asks) != 0 { - t.Fatalf("held = %+v, steps %d, asks %d", p.Held, len(p.Steps), len(asks.asks)) - } - // The runner's own pipeline is never held: it runs on the running runner, so its pass lets the new runner be wanted (A-328). - if p, _, _ := store.Get("main.4.runner"); p.Held != nil || p.Outcome != seat.PipelineOutcomeRunning { - t.Errorf("the runner's own pipeline is held %+v, want it running on the running runner", p.Held) - } - if strings.Join(said, "|") != "wait-held.app.main.c1|pipeline-started.runner.main.c2" || len(bounds) != 1 || bounds[0] != "src2:new" { - t.Errorf("said %v, bounds %v", said, bounds) - } - // A later take-in of the runner's own module while one is held is not held either (A-328). - if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.3", Branch: "main", Source: 6, Entries: []catalog.TakeInEntry{{Module: "runner", Commit: "c4", Fingerprint: "src2:new"}}}, view); err != nil || n != 1 { - t.Fatalf("a take-in of the runner alone = %d, %v", n, err) - } - if p, _, _ := store.Get("main.6.runner"); p.Held != nil { - t.Errorf("the runner's own later pipeline is held %+v", p.Held) - } - // A later take-in not naming the runner is held for the same version. - if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.2", Branch: "main", Source: 5, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c3", Fingerprint: "src2:bb"}}}, view); err != nil || n != 1 { - t.Fatalf("second take-in = %d, %v", n, err) - } - if p, _, _ := store.Get("main.5.app"); p.Held == nil || len(asks.asks) != 0 { - t.Fatalf("the later pipeline is not held: %+v", p.Held) - } - // Another fingerprint's release starts nothing. - if n, err := s.Release(store.memStore, "src2:other", "ran", view); err != nil || n != 0 { - t.Fatalf("release of another version = %d, %v", n, err) - } - said = nil - n, err = s.Release(store.memStore, "src2:new", "ran", view) - if err != nil || n != 2 { - t.Fatalf("release = %d, %v", n, err) - } - p, _, _ = store.Get("main.4.app") - if p.Held != nil || len(p.Steps) == 0 || len(asks.asks) == 0 { - t.Errorf("after the release: held %+v, steps %d, asks %d", p.Held, len(p.Steps), len(asks.asks)) - } - if !strings.Contains(strings.Join(said, "|"), "wait-ended.app.main.c1|pipeline-started.app.main.c1") { - t.Errorf("said %v", said) - } - if _, held := Awaiting(store.memStore)(); held { - t.Error("a pipeline is still held") - } -} - -// The verb release starts the held pipelines by hand, recorded in the hand-act log first, saying why and ending each wait -// `released`, and drops the bound of the wait; a release with no why does nothing (design 47 G §276, A-206). -func TestReleaseVerbStartsTheHeldPipelinesByHand(t *testing.T) { - reg, entries := placeFixture() - entries["runner"] = catalog.CatalogEntry{Module: "runner"} - store := memCreated{memStore{}} - asks := &stepAsks{} - var said []string - s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Self: "runner", SelfFp: "src2:old", Awaiting: Awaiting(store.memStore), - Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }, - Say: func(subject string, _ any) error { - said = append(said, subject[strings.Index(subject, ".event.")+7:]) - return nil - }} - view := func(string) (Half, bool) { return Half{Entries: entries}, true } - if _, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{ - {Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "runner", Commit: "c2", Fingerprint: "src2:new"}}}, view); err != nil { - t.Fatal(err) - } - acts, dropped := &actLog{}, []string{} - v := Verbs{Hearer: &Hearer{Store: store.memStore}, Starter: s, View: view, Acts: acts, - Drop: func(k string) error { dropped = append(dropped, k); return nil }} - if _, err := v.Release("operator", seat.ReleaseInput{}); err == nil || len(*acts) != 0 { - t.Fatalf("a release saying no why: %v, acts %v", err, *acts) - } - said = nil - a, err := v.Release("operator", seat.ReleaseInput{Why: "the new runner will not come"}) - if err != nil || a.Released != 1 { // the runner's own pipeline was never held (A-328) - t.Fatalf("release = %+v, %v", a, err) - } - if p, _, _ := store.Get("main.4.app"); p.Held != nil || len(p.Steps) == 0 || len(asks.asks) == 0 { - t.Errorf("after the release: held %+v, steps %d", p.Held, len(p.Steps)) - } - if len(*acts) != 1 || !strings.HasPrefix((*acts)[0], "mesh-pipeline-runner.release: ") || len(dropped) != 1 || dropped[0] != HeldKey("src2:new") { - t.Errorf("acts %v, dropped %v", *acts, dropped) - } - if !strings.Contains(strings.Join(said, "|"), "wait-ended.app.main.c1") { - t.Errorf("said %v", said) - } -} - -// The bound's expiry starts the held pipelines and says each as late through the starter's Late, with the awaited fingerprint and -// since when; a pipeline that stays held (no view) is not said (design 47 G §266, A-206). -func TestExpiredBoundSaysEachReleasedPipelineLate(t *testing.T) { - reg, entries := placeFixture() - store := memCreated{memStore{}} - since := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) - for _, k := range []string{"main.4.app", "dev.4.app"} { - store.memStore[k] = struct { - p seat.Pipeline - rev uint64 - }{seat.Pipeline{Pipeline: k, Branch: strings.Split(k, ".")[0], Module: "app", Outcome: seat.PipelineOutcomeRunning, - Held: &seat.Held{Since: since, Fingerprint: "src2:new"}}, 1} - } - var late []string - s := &Starter{Pipelines: store, Steps: &stepAsks{}, Name: "the-runner", Say: func(string, any) error { return nil }, - Registration: func(string) (catalog.Registration, bool) { return reg, true }, - Late: func(p seat.Pipeline, h seat.Held) { - late = append(late, p.Pipeline+"@"+h.Fingerprint+"@"+h.Since.Format(time.RFC3339)) - }} - view := func(b string) (Half, bool) { return Half{Entries: entries}, b == "main" } - if n, err := s.ReleaseExpired(store.memStore, HeldKey("src2:new"), view); err == nil || n != 1 { - t.Fatalf("release = %d, %v (the pipeline of a branch with no view stays held, an error)", n, err) - } - if len(late) != 1 || late[0] != "main.4.app@src2:new@2026-11-11T12:00:00Z" { - t.Errorf("late %v", late) - } -} - -// A wait of the step pipeline waits on the runner version (design 47 G §534, A-206). -func TestHeldWaitIsOnTheRunnerVersion(t *testing.T) { - w := (&Starter{}).waitOf(seat.Pipeline{Pipeline: "main.4.app", Branch: "main", Module: "app"}, time.Unix(1, 0), "") - if w.Step != seat.WaitStepPipeline || w.On != "runner version" { - t.Errorf("wait %+v", w) - } -} - -// The wait of a held pipeline counts from the build end of the awaited runner: wait-held says it as `clock-from` and the bound is -// armed from it; while that build has not ended nothing is armed, and hearing its end arms the bound and says wait-held again -// with `clock-from`; a failed end starts no clock (design 47 G §266, §269, A-181, A-206). -func TestHeldWaitCountsFromTheBuildEndOfTheAwaitedRunner(t *testing.T) { - reg, entries := placeFixture() - store := memCreated{memStore{}} - ends := &BuildEnds{} - var waits []seat.Wait - var armed []string - s := &Starter{Pipelines: store, Steps: &stepAsks{}, Name: "the-runner", Self: "runner", SelfFp: "src2:old", Awaiting: Awaiting(store.memStore), - Ends: ends.At, Hold: func(fp string, from time.Time) error { - armed = append(armed, fp+"@"+from.Format(time.RFC3339)) - return nil - }, - Registration: func(string) (catalog.Registration, bool) { return reg, true }, - Say: func(subject string, body any) error { - if w, ok := body.(seat.Wait); ok && strings.Contains(subject, ".wait-held.") { - waits = append(waits, w) - } - return nil - }} - view := func(string) (Half, bool) { return Half{Entries: entries}, true } - taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{ - {Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "runner", Commit: "c2", Fingerprint: "src2:new"}}} - end := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) - // The runner's build has not ended: held, no clock, no bound. - if _, err := s.TakenIn(taken, view); err != nil { - t.Fatal(err) - } - if len(waits) != 1 || waits[0].ClockFrom != nil || len(armed) != 0 { // the runner's own pipeline is never held (A-328) - t.Fatalf("waits %+v, armed %v", waits, armed) - } - // A failed end starts no clock; a done end does. - if _, _, ok := ends.Heard([]byte(`{"module":"runner","fingerprint":"src2:new","outcome":"failed"}`), end); ok { - t.Error("a failed end was kept") - } - if m, fp, ok := ends.Heard([]byte(`{"module":"runner","fingerprint":"src2:new","outcome":"done"}`), end); !ok || m != "runner" || fp != "src2:new" { - t.Fatalf("heard %q %q %v", m, fp, ok) - } - waits = nil - if n, err := s.Clocked(store.memStore, "runner", "src2:new"); err != nil || n != 1 { - t.Fatalf("clocked = %d, %v", n, err) - } - if len(waits) != 1 || waits[0].ClockFrom == nil || !waits[0].ClockFrom.Equal(end) || len(armed) != 1 || armed[0] != "src2:new@2026-11-11T12:00:00Z" { - t.Errorf("waits %+v, armed %v", waits, armed) - } - // A pipeline held after the end is known says the clock at once. - waits = nil - taken.Key, taken.Source = "m.2", 5 - if _, err := s.TakenIn(taken, view); err != nil { - t.Fatal(err) - } - if len(waits) == 0 || waits[0].ClockFrom == nil || !waits[0].ClockFrom.Equal(end) { - t.Errorf("waits %+v", waits) - } -} - -// The bound counts from the build end: the 30 minutes less what passed since, never under a second. -func TestHeldTTLCountsFromTheBuildEnd(t *testing.T) { - end := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) - for _, c := range []struct { - after, want time.Duration - }{{0, 30 * time.Minute}, {10 * time.Minute, 20 * time.Minute}, {45 * time.Minute, time.Second}} { - if got := HeldTTL(end, end.Add(c.after)); got != c.want { - t.Errorf("after %s the bound has %s, want %s", c.after, got, c.want) - } - } -} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index a01da243..b244afbb 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -107,25 +107,6 @@ func (r Reads) Stands() (seat.StandsAnswer, error) { return seat.StandsAnswer{Stands: out}, nil } -// Waits is the pipelines held for the runner's new version, each as the wait of step `pipeline` it says (design 47 G §266, §276, -// §534, A-195): the acting part awaited by its fingerprint, since when, and the pipeline. -func (r Reads) Waits() (seat.StandsAnswer, error) { - all, err := r.pipelines() - if err != nil { - return seat.StandsAnswer{}, err - } - out := []seat.Wait{} - for _, p := range all { - if p.Held == nil || p.Outcome != seat.PipelineOutcomeRunning { - continue - } - out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit, Group: p.Group, Source: p.Source, - Revision: p.Revision, Fingerprint: p.Held.Fingerprint, Step: seat.WaitStepPipeline, On: seat.WaitOnBuildNode, Since: p.Held.Since, V: 1}) - } - sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) }) - return seat.StandsAnswer{Stands: out}, nil -} - // WhyCarried is where a step's result was carried from (§25, §83): the pipeline it names. func (r Reads) WhyCarried(in seat.WhyCarriedInput) (seat.WhyCarriedAnswer, error) { p, ok, err := r.pipeline(in.Pipeline) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go index 3666f396..20c544ab 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go @@ -129,22 +129,6 @@ func TestReads(t *testing.T) { } } -// runner_waits lists the pipelines held for the runner's new version as waits of step `pipeline` (design 47 G §276, A-195). -func TestWaitsListTheHeldPipelines(t *testing.T) { - since := time.Date(2026, 11, 12, 6, 0, 0, 0, time.UTC) - held := seat.Pipeline{Pipeline: "main.4.app", Module: "app", Branch: "main", Commit: "c1", Source: 4, Outcome: seat.PipelineOutcomeRunning, - Held: &seat.Held{Since: since, Fingerprint: "src2:new"}} - plain := seat.Pipeline{Pipeline: "main.3.app", Module: "app", Branch: "main", Outcome: seat.PipelineOutcomeRunning} - pipes := memKept{} - put(t, pipes, held.Pipeline, held) - put(t, pipes, plain.Pipeline, plain) - r := Reads{Pipelines: pipes, Kept: memKept{}} - got, err := r.Waits() - if err != nil || len(got.Stands) != 1 || got.Stands[0].Step != seat.WaitStepPipeline || got.Stands[0].Fingerprint != "src2:new" || !got.Stands[0].Since.Equal(since) { - t.Fatalf("waits = %+v, %v", got, err) - } -} - // show carries each store's fill (pipelines, step-outputs, kept, step-log) and kept the kept store's, read through the bundle's state // call, never counted from the keys; a fill that cannot be read is an error, not an answer without it (design 47 G §366, §623, A-214). func TestShowAndKeptCarryTheStoresFill(t *testing.T) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cde61cce..23afc3b4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -35,8 +35,6 @@ type Begin struct { Group string Repository string Revision int64 - // Await is the fingerprint of the runner's own module this pipeline is held for (§266, A-195); empty when it is not held. - Await string // Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them. Entries map[string]catalog.CatalogEntry // Published is the package's publish that started the pipeline, nil for any other start (A-202). @@ -76,16 +74,6 @@ type Starter struct { Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error) // Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none. Arm Arm - // Self is the runner's own module and SelfFp the fingerprint it runs (A-149); a take-in giving Self another fingerprint - // holds its pipelines for the new runner (§65, §266, A-195). Awaiting is the fingerprint the held pipelines await, while - // any is held; Hold arms the bound of that wait (HeldBound) counting from the time the awaited build ended. All nil: nothing is held. - Self, SelfFp string - Awaiting func() (string, bool) - Hold func(fp string, from time.Time) error - // Ends says when the build of a module at a fingerprint ended (BuildEnds.At): the clock of the wait's bound (A-181, A-206). - Ends func(module, fp string) (time.Time, bool) - // Late says a pipeline the bound's expiry started with the running runner (A-206): the finding pipeline-waits. - Late func(p seat.Pipeline, held seat.Held) // Fills refuses a start while a store of the runner is full (§366); nil refuses none. Fills *Fills // Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery. @@ -124,19 +112,6 @@ func (s *Starter) Start(b Begin) (bool, error) { if err := s.Fills.refuseStart(key); err != nil { return false, err } - // The runner's own pipeline is never held: it runs on the running runner, as a build module's running version builds its - // successor (§270, A-328), so its pass lets the new runner be wanted (§29). - if b.Module != "" && b.Module == s.Self { - b.Await = "" - } - if b.Await == "" && s.Awaiting != nil && b.Module != s.Self { - if fp, ok := s.Awaiting(); ok { - b.Await = fp - } - } - if b.Await != "" { - return s.startHeld(b, key, now, b.Await) - } p := newPipeline(b, key, now) var asks []Asked if b.unheld { @@ -289,7 +264,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe note(fmt.Errorf("the pipeline %s is not read: %w", key, err)) continue } - if p.Outcome != seat.PipelineOutcomeRunning || p.Held != nil || p.Run != "" || len(p.Steps) > 0 { + if p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" || len(p.Steps) > 0 { continue } r, ok := reg(p) @@ -449,17 +424,9 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half half, held := view(ev.Branch) started := 0 var first error - // A take-in giving the runner's own module a new fingerprint holds every other pipeline it starts for that runner (§65, - // A-195, A-328); the runner's own runs now (Start). - await := "" - for _, e := range ev.Entries { - if s.changesSelf(e.Module, e.Fingerprint) { - await = e.Fingerprint - } - } for _, e := range ev.Entries { ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint, - Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries, Await: await, unheld: !held}) + Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries, unheld: !held}) if err != nil && first == nil { first = err } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index ac0d68b3..619310c0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -91,6 +91,24 @@ func TestStarts(t *testing.T) { } } +// memCreated is memStore with the Create a start writes through. +type memCreated struct{ memStore } + +func (m memCreated) Create(key string, value []byte) error { + if _, ok := m.memStore[key]; ok { + return ErrExists + } + var p seat.Pipeline + if err := json.Unmarshal(value, &p); err != nil { + return err + } + m.memStore[key] = struct { + p seat.Pipeline + rev uint64 + }{p, 1} + return nil +} + type failing struct{} func (failing) Create(string, []byte) error { return errors.New("bus away") } @@ -216,4 +234,22 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T if n != 1 || err != nil || len(p.Steps) == 0 { t.Errorf("the half of next placed %d (%v): %d steps, want the publish's pipeline placed", n, err, len(p.Steps)) } + + // A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running, + // no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more. + var old seat.Pipeline + if err := json.Unmarshal([]byte(`{"pipeline":"main.940.app","branch":"main","module":"app","commit":"c9","source":940,"outcome":"running",`+ + `"held":{"since":"2026-11-15T18:05:00Z","fingerprint":"src2:new"}}`), &old); err != nil { + t.Fatal(err) + } + store.memStore["main.940.app"] = struct { + p seat.Pipeline + rev uint64 + }{old, 1} + before = len(asks.asks) + n, err = s.Waiting(store.memStore, "main", late) + p, _, _ = store.Get("main.940.app") + if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == before { + t.Errorf("the pipeline a runner before this one set aside: placed %d (%v), %d steps, %d asks more, want it started as a newly started one", n, err, len(p.Steps), len(asks.asks)-before) + } } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 784a16b3..80d88e8a 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -76,9 +76,9 @@ func TestManifest(t *testing.T) { {"sent-objects 256 MiB", objectMiB[catalogue.SentObjectsLocal] == 256}, // §333, §54, §44, §53, §376, §24, §37, §4, §77: what the runner hears. {"consumes", equalSets(m.Consumes, []string{ - "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "mesh-delivery-builder.build.ended.**", + "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", - "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "node-engine.running", "mesh-wants.step-ended", "mesh-catalog.registration-changed", + "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})}, @@ -107,7 +107,7 @@ func TestManifest(t *testing.T) { "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})}, {"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})}, // RN25 (§539): the runner's tools carry the prefix runner_. - {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_waits", "runner_release", "runner_unlogged", "runner_unlogged_append"})}, + {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})}, // A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery, // never by a tool of its own; runner_health stays a tool. {"health is the unit", healthTool(raw) == "unit"}, diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 5a61a86c..9198f2fa 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -18,8 +18,7 @@ "rerun", "accept", "cancel", - "ask-again", - "release" + "ask-again" ] } ], @@ -27,8 +26,6 @@ "runner_first_views", "runner_first_view_again", "runner_health", - "runner_waits", - "runner_release", "runner_unlogged", "runner_unlogged_append" ], @@ -37,7 +34,6 @@ "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", - "mesh-delivery-builder.build.ended.**", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", @@ -45,7 +41,6 @@ "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", - "node-engine.running", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision",