pipeline runner: no pipeline is set aside for the runner's own new version
A take-in that changes the runner starts its pipelines as any take-in does; the state bounds' held keys, the wait's `on` value `runner version`, the finding pipeline-waits, the verb release (and the tools runner_release and runner_waits that only served it), the Held field and the runner's build-end and running hearing go. A pipeline a previous runner left set aside is placed and asked at the runner's start as a newly started one. Builds: design 47 G A-446 (replaces A-195, A-206, A-218, §266's hold)
This commit is contained in:
@@ -27,7 +27,6 @@ const (
|
||||
VerbAccept = "accept"
|
||||
VerbCancel = "cancel"
|
||||
VerbAskAgain = "ask-again"
|
||||
VerbRelease = "release"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -46,7 +45,7 @@ const (
|
||||
|
||||
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
|
||||
func Verbs() []string {
|
||||
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain, VerbRelease}
|
||||
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain}
|
||||
}
|
||||
|
||||
// Events is every event published on the seat's subject: a new copy each call.
|
||||
@@ -360,14 +359,6 @@ const (
|
||||
GroupEndedOutcomeCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// Held is a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195).
|
||||
type Held struct {
|
||||
// the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195)
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
// since when the pipeline is held, RFC 3339
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// KeptResult is a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8).
|
||||
type KeptResult struct {
|
||||
// sha256:<hex> of what the package's tests read
|
||||
@@ -452,7 +443,6 @@ type Pipeline struct {
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
// the gathered group, when the pipeline is its
|
||||
Group string `json:"group,omitempty"`
|
||||
Held *Held `json:"held,omitempty"`
|
||||
Module string `json:"module"`
|
||||
Outcome string `json:"outcome"`
|
||||
// yes when the module is over its test budget (design 47 G section 419)
|
||||
@@ -731,7 +721,7 @@ type Wait struct {
|
||||
// for its end, how it ended: answered, ran, released, superseded
|
||||
How string `json:"how,omitempty"`
|
||||
Module string `json:"module"`
|
||||
// what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206)
|
||||
// what the step waits on (design 47 *Health and going back*)
|
||||
On string `json:"on"`
|
||||
// the pipeline's key, <branch>.<revision>.<module>, a branch escaped as a key's name is
|
||||
Pipeline string `json:"pipeline"`
|
||||
@@ -751,10 +741,9 @@ type Wait struct {
|
||||
|
||||
// The values of Wait's on.
|
||||
const (
|
||||
WaitOnPerson = "person"
|
||||
WaitOnBuildNode = "build node"
|
||||
WaitOnRevisionView = "revision view"
|
||||
WaitOnRunnerVersion = "runner version"
|
||||
WaitOnPerson = "person"
|
||||
WaitOnBuildNode = "build node"
|
||||
WaitOnRevisionView = "revision view"
|
||||
)
|
||||
|
||||
// The values of Wait's step.
|
||||
@@ -991,18 +980,6 @@ type AskAgainAnswer struct {
|
||||
Result StepResult `json:"result"`
|
||||
}
|
||||
|
||||
// ReleaseInput is what release takes.
|
||||
type ReleaseInput struct {
|
||||
// why, written with the caller the runtime proves in the hand-act log
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// ReleaseAnswer is what release answers.
|
||||
type ReleaseAnswer struct {
|
||||
// how many held pipelines were started
|
||||
Released int64 `json:"released"`
|
||||
}
|
||||
|
||||
// PipelineStartedEvent is the body of the event pipeline-started.
|
||||
type PipelineStartedEvent = PipelineStarted
|
||||
|
||||
|
||||
@@ -501,35 +501,6 @@
|
||||
]
|
||||
},
|
||||
"class": "act"
|
||||
},
|
||||
{
|
||||
"name": "release",
|
||||
"description": "The pipelines the runner holds for its new version started by hand with the running runner, whatever the new version's state; written in the hand-act log (design 47 G 276, A-206).",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"why": {
|
||||
"type": "string",
|
||||
"description": "why, written with the caller the runtime proves in the hand-act log"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"why"
|
||||
]
|
||||
},
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"released": {
|
||||
"type": "integer",
|
||||
"description": "how many held pipelines were started"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"released"
|
||||
]
|
||||
},
|
||||
"class": "act"
|
||||
}
|
||||
],
|
||||
"events": [
|
||||
@@ -719,25 +690,6 @@
|
||||
},
|
||||
"description": "how many tests a step ran and carried, per level (ADR 0666 section 8)"
|
||||
},
|
||||
"Held": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"since": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "since when the pipeline is held, RFC 3339"
|
||||
},
|
||||
"fingerprint": {
|
||||
"type": "string",
|
||||
"description": "the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"since",
|
||||
"fingerprint"
|
||||
],
|
||||
"description": "a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195)"
|
||||
},
|
||||
"ToolchainUsed": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -1074,9 +1026,6 @@
|
||||
"type": "boolean",
|
||||
"description": "yes when the module is over its test budget (design 47 G section 419)"
|
||||
},
|
||||
"held": {
|
||||
"$ref": "#/defs/Held"
|
||||
},
|
||||
"widens": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -1537,10 +1486,9 @@
|
||||
"enum": [
|
||||
"person",
|
||||
"build node",
|
||||
"revision view",
|
||||
"runner version"
|
||||
"revision view"
|
||||
],
|
||||
"description": "what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206)"
|
||||
"description": "what the step waits on (design 47 *Health and going back*)"
|
||||
},
|
||||
"who": {
|
||||
"type": "string",
|
||||
|
||||
@@ -17,7 +17,6 @@ export declare const verbs: {
|
||||
readonly accept: "accept";
|
||||
readonly cancel: "cancel";
|
||||
readonly askAgain: "ask-again";
|
||||
readonly release: "release";
|
||||
};
|
||||
|
||||
export declare const events: {
|
||||
@@ -154,14 +153,6 @@ export interface GroupEnded {
|
||||
"why"?: string;
|
||||
}
|
||||
|
||||
/** a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195) */
|
||||
export interface Held {
|
||||
/** the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195) */
|
||||
"fingerprint": string;
|
||||
/** since when the pipeline is held, RFC 3339 */
|
||||
"since": string;
|
||||
}
|
||||
|
||||
/** a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8) */
|
||||
export interface KeptResult {
|
||||
/** sha256:<hex> of what the package's tests read */
|
||||
@@ -211,7 +202,6 @@ export interface Pipeline {
|
||||
"fingerprint"?: string;
|
||||
/** the gathered group, when the pipeline is its */
|
||||
"group"?: string;
|
||||
"held"?: Held;
|
||||
"module": string;
|
||||
"outcome": "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
|
||||
/** yes when the module is over its test budget (design 47 G section 419) */
|
||||
@@ -414,8 +404,8 @@ export interface Wait {
|
||||
/** for its end, how it ended: answered, ran, released, superseded */
|
||||
"how"?: string;
|
||||
"module": string;
|
||||
/** what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206) */
|
||||
"on": "person" | "build node" | "revision view" | "runner version";
|
||||
/** what the step waits on (design 47 *Health and going back*) */
|
||||
"on": "person" | "build node" | "revision view";
|
||||
/** the pipeline's key, <branch>.<revision>.<module>, a branch escaped as a key's name is */
|
||||
"pipeline": string;
|
||||
/** the catalog graph revision of the branch the pipeline runs for */
|
||||
@@ -606,18 +596,6 @@ export interface AskAgainAnswer {
|
||||
"result": StepResult;
|
||||
}
|
||||
|
||||
/** is what release takes. */
|
||||
export interface ReleaseInput {
|
||||
/** why, written with the caller the runtime proves in the hand-act log */
|
||||
"why": string;
|
||||
}
|
||||
|
||||
/** is what release answers. */
|
||||
export interface ReleaseAnswer {
|
||||
/** how many held pipelines were started */
|
||||
"released": number;
|
||||
}
|
||||
|
||||
/** is the body of the event pipeline-started. */
|
||||
export type PipelineStartedEvent = PipelineStarted;
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ export const verbs = {
|
||||
accept: "accept",
|
||||
cancel: "cancel",
|
||||
askAgain: "ask-again",
|
||||
release: "release",
|
||||
};
|
||||
|
||||
export const events = {
|
||||
|
||||
@@ -107,34 +107,6 @@ func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time) stdio.
|
||||
Run: func(map[string]any) (any, error) { return pipeline.Health(pipelines, consumers, now()), nil }}
|
||||
}
|
||||
|
||||
// WaitsTool is the runner's tool runner_waits: the pipelines held for its new version, with the fingerprint each awaits and since
|
||||
// when (design 47 G §276, §539, A-195).
|
||||
func WaitsTool(r pipeline.Reads) stdio.Tool {
|
||||
return stdio.Tool{Name: Prefix + "_waits",
|
||||
Description: "The pipelines the pipeline runner holds until its new version runs healthy: each with its pipeline, the " +
|
||||
"fingerprint of the runner it awaits and since when.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return r.Waits() }}
|
||||
}
|
||||
|
||||
// ReleaseTool is the runner's tool runner_release: it starts the pipelines held for the runner's new version with the running
|
||||
// runner, through the seat's verb release, which the process that holds the pipelines answers and records in the hand-act log
|
||||
// first (design 47 G §276, A-206).
|
||||
func ReleaseTool(ask func(key string, body any) (json.RawMessage, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: Prefix + "_release",
|
||||
Description: "Start now, with the running pipeline runner, the pipelines it holds until its new version runs healthy " +
|
||||
"(runner_waits lists them), each wait ending released. Written in the hand-act log (mesh-hand-acts).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
in, err := as[seat.ReleaseInput](args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ask("seat:"+seat.Seat+"."+seat.VerbRelease, in)
|
||||
}}
|
||||
}
|
||||
|
||||
// StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states
|
||||
// pipelines and kept by `mesh/state.info`, step-outputs by the object store's fill and step-log by the log's; the runner's
|
||||
// process writes nothing for it. A store whose fill cannot be read is an error naming it.
|
||||
@@ -177,7 +149,6 @@ func main() {
|
||||
func(n string) (*stdio.LogInfo, error) { return stdio.Log(n).Info() }, time.Now)
|
||||
r := pipeline.Reads{Pipelines: kept{stdio.State(seat.StatePipelines)}, Kept: kept{stdio.State(seat.StateKept)}, Fill: fill}
|
||||
tools := append(Tools(r), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
|
||||
tools = append(tools, WaitsTool(r), ReleaseTool(stdio.Ask))
|
||||
tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask},
|
||||
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...)
|
||||
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -34,21 +33,11 @@ func TestTools(t *testing.T) {
|
||||
}
|
||||
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
|
||||
names = append(names, HealthTool(none{}, none{}, nil).Name)
|
||||
// A-206: the tool runner_release asks the seat's verb release with its why.
|
||||
var asked string
|
||||
rel := ReleaseTool(func(key string, body any) (json.RawMessage, error) {
|
||||
asked = key + fmt.Sprintf(" %+v", body)
|
||||
return json.RawMessage(`{"released":1}`), nil
|
||||
})
|
||||
if _, err := rel.Run(map[string]any{"why": "w"}); err != nil || asked != "seat:mesh-pipeline-runner.release {Why:w}" {
|
||||
t.Errorf("runner_release asked %q, %v", asked, err)
|
||||
}
|
||||
names = append(names, rel.Name)
|
||||
// §606, §549: the unlogged entries are listed and appended by hand.
|
||||
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
|
||||
names = append(names, tl.Name)
|
||||
}
|
||||
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_health runner_release runner_unlogged runner_unlogged_append"
|
||||
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_health runner_unlogged runner_unlogged_append"
|
||||
if got := strings.Join(names, " "); got != want {
|
||||
t.Errorf("tools %q, want %q", got, want)
|
||||
}
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// hearBuildEnds hears the builder's stream of build ends, kept last per subject and read whole at the start, then followed
|
||||
// (design 47 G A-181, §556): when the build of the runner's new version ended is the time the wait's bound counts from, and
|
||||
// each end heard for a runner held pipelines await starts their clock (§266, A-206). A stream it cannot hear is said, and the
|
||||
// bound of such a wait is not armed until the runner starts again.
|
||||
func hearBuildEnds(ctx context.Context, cred process.Account, js jetstream.JetStream, ends *pipeline.BuildEnds, st *pipeline.Starter, store pipeline.Updater,
|
||||
logf func(string, ...any)) {
|
||||
stream := catalogue.StreamBusName(builds.BuildsStream)
|
||||
cc, err := cred.HearStream(ctx, js, stream, func(m jetstream.Msg) {
|
||||
defer m.Ack()
|
||||
md, err := m.Metadata()
|
||||
if err != nil {
|
||||
logf("%s has no time the stream stamped: %v", m.Subject(), err)
|
||||
return
|
||||
}
|
||||
module, fp, ok := ends.Heard(m.Data(), md.Timestamp)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if n, err := st.Clocked(store, module, fp); err != nil {
|
||||
logf("the clock of the %d pipelines held for %s: %v", n, fp, err)
|
||||
} else if n > 0 {
|
||||
logf("the build of %s ended: the bound of the %d pipelines held for it counts from it", fp, n)
|
||||
}
|
||||
}, logf)
|
||||
if err != nil {
|
||||
logf("the stream %s is not heard, so no wait for the runner's new version has its bound armed until the runner starts again: %v", stream, err)
|
||||
return
|
||||
}
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
}
|
||||
-76
@@ -1,76 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// The runner hears the builder's stream of build ends, kept last per subject, read whole at its start: the end of its new version's
|
||||
// build, published before it started, starts the clock of the pipelines held for it, counted from the time the stream stamped the end
|
||||
// (design 47 G A-181, §266, A-206).
|
||||
func TestTheRunnerHearsTheBuildEndOfItsNewVersionAtItsStart(t *testing.T) {
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
stream, subject := catalogue.StreamBusName(builds.BuildsStream), process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
published := time.Now().UTC()
|
||||
if _, err := js.Publish(ctx, process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEventName("mesh-delivery-pipeline", "src2:new")),
|
||||
[]byte(`{"module":"mesh-delivery-pipeline","fingerprint":"src2:new","outcome":"done"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cred := process.Account{Node: "n1", Module: "mesh-delivery-pipeline"}
|
||||
durable := cred.SeatEventConsumerConfig([]string{subject}, true)
|
||||
durable.DeliverPolicy = jetstream.DeliverNewPolicy
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, durable); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "pipelines", History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := pipelineStore{ctx: ctx, kv: kv}
|
||||
held, _ := json.Marshal(seat.Pipeline{Pipeline: "main.4.app", Branch: "main", Module: "app", Outcome: seat.PipelineOutcomeRunning,
|
||||
Held: &seat.Held{Since: published, Fingerprint: "src2:new"}})
|
||||
if _, err := kv.Create(ctx, "main.4.app", held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ends := &pipeline.BuildEnds{}
|
||||
armed := make(chan time.Time, 1)
|
||||
st := &pipeline.Starter{Self: "mesh-delivery-pipeline", Ends: ends.At, Say: func(string, any) error { return nil },
|
||||
Hold: func(fp string, from time.Time) error { armed <- from; return nil }}
|
||||
hearBuildEnds(ctx, cred, js, ends, st, store, t.Logf)
|
||||
select {
|
||||
case from := <-armed:
|
||||
if from.Before(published.Add(-time.Minute)) || from.After(time.Now().Add(time.Second)) {
|
||||
t.Errorf("the bound counts from %s, not from the end published at %s", from, published)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the end of the new runner's build was not heard at the start")
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -12,34 +9,11 @@ import (
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// findingsWriter is the runner's finding writer as the mesh-sdk makes it.
|
||||
type findingsWriter = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]
|
||||
|
||||
// KindPipelineWaits is the finding of pipelines held for the runner's new version that the wait's bound started with the running
|
||||
// runner (design 47 G §266, A-206).
|
||||
const KindPipelineWaits = "pipeline-waits"
|
||||
|
||||
// awaitedSaid opens the evidence entry of a pipeline-waits finding that names the awaited runner fingerprint, `awaited <fp>`
|
||||
// (A-276): the runner clears a finding by that entry alone, never by its summary.
|
||||
const awaitedSaid = "awaited "
|
||||
|
||||
// lateFinding is the finding pipeline-waits of one pipeline the bound's expiry started: scope module, id its key
|
||||
// `<branch>.<id>.<module>`, naming the fingerprint of the new runner it waited for and since when, the fingerprint also in one
|
||||
// evidence entry `awaited <fp>` (A-276).
|
||||
func lateFinding(p seat.Pipeline, held seat.Held, runner, node string, at time.Time) condseat.Finding {
|
||||
return condseat.Finding{Scope: views.FindingScope, ID: p.Pipeline, Kind: KindPipelineWaits, Severity: condseat.FindingSeverityWarning,
|
||||
Evidence: []condseat.Evidence{{At: at, Said: awaitedSaid + held.Fingerprint}},
|
||||
Headline: "the new pipeline runner did not run healthy in time",
|
||||
Summary: fmt.Sprintf("the pipeline %s was held since %s for the runner at %s to run healthy; the bound passed and the running runner runs it",
|
||||
p.Pipeline, held.Since.UTC().Format(time.RFC3339), held.Fingerprint),
|
||||
Source: "bound of the wait for the new runner", Observer: runner, At: at,
|
||||
Subject: condseat.Subject{Scope: views.FindingScope, ID: p.Module, Machine: node}}
|
||||
}
|
||||
|
||||
// findingWriter is the runner's mesh-sdk held finding writer (design 47 G §461, §689, A-29): the runner raises its findings
|
||||
// as they happen (a failed head, §33), so the writer holds what it has written and its statement says exactly that, with the
|
||||
// reader's open findings. The writer
|
||||
@@ -58,33 +32,3 @@ func findingWriter(ctx context.Context, cred process.Account, conn *nats.Conn, j
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// heldFindings is what clearAwaited needs of the runner's held finding writer: what it holds and a clear.
|
||||
type heldFindings interface {
|
||||
Held() []condseat.Finding
|
||||
Clear(ctx context.Context, scope, id, kind, why string) error
|
||||
}
|
||||
|
||||
// clearAwaited clears exactly the pipeline-waits findings whose evidence names fp awaited (A-218, A-276): the node-engine seat's
|
||||
// running says the runner at fp runs healthy. The writer holds what its state findings hold from its start, so a finding raised
|
||||
// before a restart is cleared too. It says how many it cleared, and the first clear that failed.
|
||||
func clearAwaited(ctx context.Context, w heldFindings, fp string) (int, error) {
|
||||
if w == nil || fp == "" {
|
||||
return 0, nil
|
||||
}
|
||||
n := 0
|
||||
var first error
|
||||
for _, f := range w.Held() {
|
||||
if f.Kind != KindPipelineWaits || !slices.ContainsFunc(f.Evidence, func(e condseat.Evidence) bool { return e.Said == awaitedSaid+fp }) {
|
||||
continue
|
||||
}
|
||||
if err := w.Clear(ctx, f.Scope, f.ID, f.Kind, "the runner at "+fp+" runs healthy"); err != nil {
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n, first
|
||||
}
|
||||
|
||||
-110
@@ -1,110 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// Design 47 G A-218, A-276: a pipeline-waits finding raised before the runner restarted is held again by the new process's writer,
|
||||
// seeded from the state findings, and is cleared when the node-engine seat's running says the awaited runner fingerprint healthy;
|
||||
// the finding awaiting another fingerprint stands.
|
||||
func TestAPipelineWaitsFindingIsClearedAfterARestartWhenItsAwaitedRunnerRunsHealthy(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
const module = "mesh-delivery-pipeline"
|
||||
states := map[string]jetstream.KeyValue{}
|
||||
for _, state := range []string{"findings", "pipelines", "bounds"} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
states[state] = kv
|
||||
}
|
||||
// Where the writer says its findings, as the bus keeps the module's events.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cred := process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"}
|
||||
conn, err := process.Connect(cred)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
since := time.Now().UTC().Add(-time.Hour)
|
||||
before, err := findingWriter(ctx, cred, nc, js, conn.State("findings"), t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range []struct{ key, module, fp string }{{"main.4.app", "app", "src2:new"}, {"main.5.web", "web", "src2:newer"}} {
|
||||
if err := before.Write(ctx, lateFinding(seat.Pipeline{Pipeline: p.key, Module: p.module, Branch: "main"},
|
||||
seat.Held{Since: since, Fingerprint: p.fp}, module, "n1", since.Add(30*time.Minute))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before.Close()
|
||||
keys := func() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
ks, err := states["findings"].Keys(ctx)
|
||||
if err != nil && err != jetstream.ErrNoKeysFound {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range ks {
|
||||
out[k] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
app, web := "module.main.4.app.pipeline-waits", "module.main.5.web.pipeline-waits"
|
||||
if k := keys(); !k[app] || !k[web] {
|
||||
t.Fatalf("the state findings holds %v before the restart, want %s and %s", k, app, web)
|
||||
}
|
||||
|
||||
// The restart: a new process's writer holds what its state findings hold.
|
||||
after, err := findingWriter(ctx, cred, nc, js, conn.State("findings"), t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer after.Close()
|
||||
if len(after.Held()) != 2 {
|
||||
t.Fatalf("the writer after the restart holds %+v, want both findings", after.Held())
|
||||
}
|
||||
hub := &oneHub{}
|
||||
starter := &pipeline.Starter{Logf: t.Logf}
|
||||
if err := hearRunning(ctx, hub, module, starter, pipelineStore{ctx: ctx, kv: states["pipelines"]}, states["bounds"], after,
|
||||
func(string) (pipeline.Half, bool) { return pipeline.Half{}, false }, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healthy := true
|
||||
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: "n1", Module: module, Fingerprint: "src2:new", Healthy: &healthy, At: time.Now().UTC()})
|
||||
var acks []string
|
||||
hub.handle(dataMsg{data: raw, acks: &acks})
|
||||
if k := keys(); k[app] || !k[web] {
|
||||
t.Errorf("after the runner at src2:new runs healthy the state findings holds %v: want %s cleared and %s standing", k, app, web)
|
||||
}
|
||||
if held := after.Held(); len(held) != 1 || held[0].ID != "main.5.web" {
|
||||
t.Errorf("the writer holds %+v, want only the finding awaiting src2:newer", held)
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// The bound's expiry's finding is pipeline-waits, scope module, id the pipeline's key, naming the awaited fingerprint and since
|
||||
// when (design 47 G §266, A-206); it passes the writer's own check of a finding.
|
||||
func TestLateFindingIsPipelineWaitsOfTheModuleScope(t *testing.T) {
|
||||
since := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
|
||||
f := lateFinding(seat.Pipeline{Pipeline: "main.4.app", Module: "app", Branch: "main"}, seat.Held{Since: since, Fingerprint: "src2:new"},
|
||||
"mesh-delivery-pipeline", "n1", since.Add(30*time.Minute))
|
||||
if f.Kind != "pipeline-waits" || f.Scope != views.FindingScope || f.ID != "main.4.app" || f.Severity != condseat.FindingSeverityWarning {
|
||||
t.Errorf("finding %+v", f)
|
||||
}
|
||||
if !strings.Contains(f.Summary, "src2:new") || !strings.Contains(f.Summary, "2026-11-11T12:00:00Z") {
|
||||
t.Errorf("summary %q", f.Summary)
|
||||
}
|
||||
if err := views.CheckFinding(f); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
// A-276: the awaited fingerprint is in one evidence entry, `awaited <fp>`.
|
||||
if len(f.Evidence) != 1 || f.Evidence[0].Said != "awaited src2:new" {
|
||||
t.Errorf("evidence %+v, want one entry `awaited src2:new`", f.Evidence)
|
||||
}
|
||||
// A-218, A-276: a healthy running of the runner at a fingerprint clears exactly the pipeline-waits findings whose evidence
|
||||
// names it awaited: not one awaiting another, not another kind naming it, never by the summary.
|
||||
other := lateFinding(seat.Pipeline{Pipeline: "main.5.web", Module: "web"}, seat.Held{Since: since, Fingerprint: "src2:newer"}, "r", "n1", since)
|
||||
other.Summary = f.Summary // a summary naming src2:new is never read
|
||||
kind := f
|
||||
kind.Kind, kind.ID = "step-failed", "main.4.app.build"
|
||||
held := &memFindings{held: []condseat.Finding{f, other, kind}}
|
||||
if n, err := clearAwaited(context.Background(), held, "src2:new"); err != nil || n != 1 || !slices.Equal(held.cleared, []string{"module.main.4.app.pipeline-waits"}) {
|
||||
t.Errorf("cleared %d (%v): %v, want the one pipeline-waits awaiting src2:new", n, err, held.cleared)
|
||||
}
|
||||
}
|
||||
|
||||
// memFindings is a held finding writer's Held and Clear.
|
||||
type memFindings struct {
|
||||
held []condseat.Finding
|
||||
cleared []string
|
||||
}
|
||||
|
||||
func (m *memFindings) Held() []condseat.Finding { return m.held }
|
||||
func (m *memFindings) Clear(_ context.Context, scope, id, kind, _ string) error {
|
||||
m.cleared = append(m.cleared, scope+"."+id+"."+kind)
|
||||
return nil
|
||||
}
|
||||
@@ -137,20 +137,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
|
||||
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
|
||||
// for its step.
|
||||
ends := &pipeline.BuildEnds{}
|
||||
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
|
||||
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
|
||||
Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
// The runner's own module and fingerprint (A-149): a take-in giving it another holds the pipelines it starts (§65, A-195).
|
||||
Self: cred.Module, SelfFp: getenv("MESH_FINGERPRINT"), Awaiting: pipeline.Awaiting(pipelineStore{ctx: ctx, kv: pipelines}),
|
||||
Ends: ends.At, Hold: armHeld(ctx, bounds),
|
||||
// The bound's expiry writes the finding pipeline-waits for each pipeline it started (§266, A-206).
|
||||
Late: func(p seat.Pipeline, held seat.Held) {
|
||||
module, node := rv.Findings.ModuleNode()
|
||||
if err := rv.Findings.Write(ctx, lateFinding(p, held, module, node, time.Now().UTC())); err != nil {
|
||||
logger.Printf("the finding pipeline-waits of %s is not written: %v", p.Pipeline, err)
|
||||
}
|
||||
},
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has}
|
||||
placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error {
|
||||
n, err := starter.PlaceRun(ev, half, rv.Catalog.View)
|
||||
@@ -162,10 +151,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// Pipelines held for the fingerprint this runner runs are started now: it is the new runner (§19, A-195).
|
||||
if n, err := starter.Release(pipelineStore{ctx: ctx, kv: pipelines}, starter.SelfFp, "ran", rv.Catalog.View); err != nil {
|
||||
logger.Printf("the pipelines held for the runner at %s: %d released, then %v", starter.SelfFp, n, err)
|
||||
}
|
||||
// A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327).
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err)
|
||||
@@ -234,23 +219,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
store := pipelineStore{ctx: ctx, kv: pipelines}
|
||||
expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
if err := watchBounds(ctx, bounds, func(key string) {
|
||||
if !pipeline.IsHeldKey(key) {
|
||||
expired(key)
|
||||
return
|
||||
}
|
||||
if n, err := starter.ReleaseExpired(store, key, rv.Catalog.View); err != nil {
|
||||
logger.Printf("the pipelines held for the runner, whose %s passed: %d released, then %v", key, n, err)
|
||||
} else {
|
||||
logger.Printf("the runner's new version did not run healthy within %s: %d held pipelines started with the running runner", pipeline.HeldBound, n)
|
||||
}
|
||||
}, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
hearBuildEnds(ctx, cred, js, ends, starter, store, logger.Printf)
|
||||
if err := hearRunning(ctx, heard, cred.Module, starter, store, bounds, rv.Findings, rv.Catalog.View, logger.Printf); err != nil {
|
||||
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil {
|
||||
|
||||
@@ -6,11 +6,9 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
@@ -178,61 +176,6 @@ func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm {
|
||||
}
|
||||
}
|
||||
|
||||
// armHeld is the bound of the pipelines held for the runner's new version: a key of the state bounds named for the awaited
|
||||
// fingerprint, with its own time to live of 30 minutes counted from the end of that runner's build (design 47 G §266, A-195, A-77,
|
||||
// A-206).
|
||||
func armHeld(ctx context.Context, bounds jetstream.KeyValue) func(string, time.Time) error {
|
||||
return func(fp string, from time.Time) error {
|
||||
_, err := bounds.Create(ctx, pipeline.HeldKey(fp), []byte(`{"fingerprint":"`+fp+`"}`), jetstream.KeyTTL(pipeline.HeldTTL(from, time.Now())))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// hearRunning releases the pipelines held for the runner's new version when the node-engine seat says the runner's own module
|
||||
// runs that fingerprint healthy, on any node (§266, §276, A-195); its bound is then deleted, and the pipeline-waits findings
|
||||
// that await it are cleared (A-218, A-276). A running with no health reads as healthy (§520). Heard again when the release or
|
||||
// a clear could not be written.
|
||||
func hearRunning(ctx context.Context, hub handlers, self string, st *pipeline.Starter, store pipeline.Updater, bounds jetstream.KeyValue,
|
||||
findings heldFindings, view func(string) (pipeline.Half, bool), logf func(string, ...any)) error {
|
||||
return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) {
|
||||
var ev nodeengine.Running
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil {
|
||||
logf("%s is no running: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if ev.Module != self || ev.Stopped || ev.Fingerprint == "" || (ev.Healthy != nil && !*ev.Healthy) {
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
n, err := st.Release(store, ev.Fingerprint, "ran", view)
|
||||
if err != nil {
|
||||
logf("the pipelines held for %s: %d released, then %v; heard again", ev.Fingerprint, n, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
logf("the runner runs %s on %s: %d held pipelines released", ev.Fingerprint, ev.Node, n)
|
||||
if err := bounds.Delete(ctx, pipeline.HeldKey(ev.Fingerprint)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
logf("the bound of the wait for %s is not deleted: %v", ev.Fingerprint, err)
|
||||
}
|
||||
}
|
||||
cleared, err := clearAwaited(ctx, findings, ev.Fingerprint)
|
||||
if cleared > 0 {
|
||||
logf("the runner runs %s on %s: %d pipeline-waits findings cleared", ev.Fingerprint, ev.Node, cleared)
|
||||
}
|
||||
if err != nil {
|
||||
logf("a pipeline-waits finding awaiting %s is not cleared: %v; heard again", ev.Fingerprint, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
})
|
||||
}
|
||||
|
||||
// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32,
|
||||
// §535, A-77), and says a drop no holder answered unanswered (A-222): no timer in the process.
|
||||
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
|
||||
|
||||
@@ -41,8 +41,7 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu
|
||||
return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v))
|
||||
}
|
||||
|
||||
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, ask-again and
|
||||
// release (§64, §306; RN12, RN14; A-206).
|
||||
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again.
|
||||
func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }),
|
||||
@@ -52,7 +51,6 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
|
||||
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
|
||||
seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }),
|
||||
seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }),
|
||||
seat.VerbRelease: verbOf(func(c string, in seat.ReleaseInput) (any, error) { return v.Release(c, in) }),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) {
|
||||
got = append(got, v)
|
||||
}
|
||||
slices.Sort(got)
|
||||
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbRelease, seat.VerbReview, seat.VerbStart, seat.VerbRun}
|
||||
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun}
|
||||
slices.Sort(want)
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("the process answers %v, want %v", got, want)
|
||||
|
||||
@@ -9,13 +9,11 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BuildEnds are the done builds' ends heard from the builder's stream of build ends (event build.ended, kept last per subject
|
||||
// and read whole at the start, design 47 G A-181): when each build, by module and fingerprint, ended, which is the time a held
|
||||
// wait's bound counts from (§266, A-206).
|
||||
type BuildEnds struct {
|
||||
mu sync.Mutex
|
||||
at map[string]time.Time
|
||||
}
|
||||
|
||||
// buildEnd is the part of the body of build.ended the runner reads.
|
||||
type buildEnd struct {
|
||||
Module string `json:"module"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Outcome string `json:"outcome"`
|
||||
}
|
||||
|
||||
// Heard keeps the end a message of the stream says, at the time the stream stamped it; a failed end, or a body naming no module
|
||||
// and fingerprint, is kept for nothing. It says the module and fingerprint it kept.
|
||||
func (e *BuildEnds) Heard(data []byte, at time.Time) (module, fp string, ok bool) {
|
||||
var b buildEnd
|
||||
if err := json.Unmarshal(data, &b); err != nil || b.Module == "" || b.Fingerprint == "" || b.Outcome == "failed" {
|
||||
return "", "", false
|
||||
}
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if e.at == nil {
|
||||
e.at = map[string]time.Time{}
|
||||
}
|
||||
e.at[b.Module+"\x00"+b.Fingerprint] = at.UTC()
|
||||
return b.Module, b.Fingerprint, true
|
||||
}
|
||||
|
||||
// At is when the build of the module at the fingerprint ended, false when no end was heard.
|
||||
func (e *BuildEnds) At(module, fp string) (time.Time, bool) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
t, ok := e.at[module+"\x00"+fp]
|
||||
return t, ok
|
||||
}
|
||||
@@ -1,269 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// HeldBound is how long the pipelines held for the runner's new version wait for it to run healthy (design 47 G §266: 30
|
||||
// minutes for the runner's `pipeline` step, A-195): a key of the state `bounds` with its own time to live, never a timer.
|
||||
const HeldBound = 30 * time.Minute
|
||||
|
||||
// HeldPrefix starts the key of a held wait's bound.
|
||||
const HeldPrefix = "held-"
|
||||
|
||||
// HeldKey is the bound's key for the awaited fingerprint: a fingerprint holds characters a key's name does not.
|
||||
func HeldKey(fp string) string {
|
||||
sum := sha256.Sum256([]byte(fp))
|
||||
return HeldPrefix + hex.EncodeToString(sum[:12])
|
||||
}
|
||||
|
||||
// IsHeldKey says whether a key of the state `bounds` is a held wait's bound.
|
||||
func IsHeldKey(key string) bool {
|
||||
return len(key) > len(HeldPrefix) && key[:len(HeldPrefix)] == HeldPrefix
|
||||
}
|
||||
|
||||
// Awaiting is the awaited fingerprint of the pipelines held in store, "" and false when none is held (design 47 G §266, A-195):
|
||||
// a pipeline started while one is held is held for the same version, so it never runs on the old runner.
|
||||
func Awaiting(store Updater) func() (string, bool) {
|
||||
return func() (string, bool) {
|
||||
pkeys, err := store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
if p, _, err := store.Get(key); err == nil && p.Held != nil && p.Outcome == seat.PipelineOutcomeRunning {
|
||||
return p.Held.Fingerprint, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// changesSelf says whether a take-in's entry is the runner's own module at a fingerprint other than the one that runs: its
|
||||
// pipelines are held for the new runner (design 47 G §65, §266, A-195).
|
||||
func (s *Starter) changesSelf(module, fp string) bool {
|
||||
return s.Self != "" && module == s.Self && fp != "" && fp != s.SelfFp
|
||||
}
|
||||
|
||||
// startHeld writes b's pipeline held for the runner's version awaiting, its steps neither placed nor asked, and says
|
||||
// wait-held with step `pipeline`; pipeline-started is said when it is released (§74, §266).
|
||||
func (s *Starter) startHeld(b Begin, key string, now time.Time, awaiting string) (bool, error) {
|
||||
p := newPipeline(b, key, now)
|
||||
p.Held = &seat.Held{Since: now, Fingerprint: awaiting}
|
||||
if err := s.create(key, p); errors.Is(err, ErrExists) {
|
||||
s.logf("%s is started already: this start writes nothing", key)
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
|
||||
}
|
||||
from := s.clockOf(awaiting)
|
||||
if from != nil && s.Hold != nil {
|
||||
if err := s.Hold(awaiting, *from); err != nil {
|
||||
s.logf("the bound of the wait for the runner at %s is not armed: %v", awaiting, err)
|
||||
}
|
||||
}
|
||||
w := s.waitOf(p, now, "")
|
||||
w.ClockFrom = from
|
||||
if err := s.Say(runnerclient.Subject(seat.EventWaitHeld, b.Module, b.Branch, commitOrGroup(p)), w); err != nil {
|
||||
s.logf("wait-held of %s is not said: %v", key, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// clockOf is when the build of the runner at fp ended, the time the bound counts from, nil while it has not (§266, A-181, A-206).
|
||||
func (s *Starter) clockOf(fp string) *time.Time {
|
||||
if s.Ends == nil {
|
||||
return nil
|
||||
}
|
||||
if at, ok := s.Ends(s.Self, fp); ok {
|
||||
return &at
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clocked starts the clock of the pipelines held for the runner at fp, the end of whose build was heard: it arms the bound from that
|
||||
// end and says wait-held again for each with `clock-from`, so the watchman times it by the bound (§269, A-206). It says how many
|
||||
// pipelines it said, and nothing when none is held for fp or the end is not known.
|
||||
func (s *Starter) Clocked(store Updater, module, fp string) (int, error) {
|
||||
if module != s.Self {
|
||||
return 0, nil
|
||||
}
|
||||
from := s.clockOf(fp)
|
||||
if from == nil {
|
||||
return 0, nil
|
||||
}
|
||||
pkeys, err := store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
said := 0
|
||||
var first error
|
||||
for _, key := range pkeys {
|
||||
p, _, err := store.Get(key)
|
||||
if err != nil || p.Held == nil || p.Outcome != seat.PipelineOutcomeRunning || p.Held.Fingerprint != fp {
|
||||
continue
|
||||
}
|
||||
if said == 0 && s.Hold != nil {
|
||||
if err := s.Hold(fp, *from); err != nil {
|
||||
first = fmt.Errorf("the bound of the wait for the runner at %s is not armed: %w", fp, err)
|
||||
}
|
||||
}
|
||||
w := s.waitOf(p, p.Held.Since, "")
|
||||
w.ClockFrom = from
|
||||
if err := s.Say(runnerclient.Subject(seat.EventWaitHeld, p.Module, p.Branch, commitOrGroup(p)), w); err != nil && first == nil {
|
||||
first = fmt.Errorf("wait-held of %s is not said: %w", key, err)
|
||||
}
|
||||
said++
|
||||
}
|
||||
return said, first
|
||||
}
|
||||
|
||||
func commitOrGroup(p seat.Pipeline) string {
|
||||
if p.Group != "" {
|
||||
return p.Group
|
||||
}
|
||||
return p.Commit
|
||||
}
|
||||
|
||||
func (s *Starter) waitOf(p seat.Pipeline, since time.Time, how string) seat.Wait {
|
||||
return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint, Group: p.Group,
|
||||
Revision: p.Revision, Source: p.Source, Step: seat.WaitStepPipeline, On: seat.WaitOnRunnerVersion, Since: since, How: how}
|
||||
}
|
||||
|
||||
// Release starts the pipelines held for fingerprint fp ("" for every held one) as the new runner running healthy lets
|
||||
// them, or the bound's expiry or a hand release (how: `ran` or `released`; §266, §276): each has its steps placed and asked and
|
||||
// says wait-ended and pipeline-started. A pipeline whose branch has no view yet stays held, and the first error is returned
|
||||
// so the event is heard again. It says how many it started.
|
||||
func (s *Starter) Release(store Updater, fp, how string, view func(branch string) (Half, bool)) (int, error) {
|
||||
pkeys, err := store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if s.Now != nil {
|
||||
now = s.Now().UTC()
|
||||
}
|
||||
started := 0
|
||||
var first error
|
||||
note := func(err error) {
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, rev, err := store.Get(key)
|
||||
if err != nil || p.Held == nil || p.Outcome != seat.PipelineOutcomeRunning || (fp != "" && p.Held.Fingerprint != fp) {
|
||||
continue
|
||||
}
|
||||
half, ok := view(p.Branch)
|
||||
if !ok {
|
||||
note(fmt.Errorf("the catalog half of %s is not held yet: %s stays held", p.Branch, key))
|
||||
continue
|
||||
}
|
||||
held := *p.Held
|
||||
p.Held = nil
|
||||
var asks []Asked
|
||||
if s.Registration != nil {
|
||||
if reg, ok := s.Registration(p.Module); ok {
|
||||
p.Steps, asks = Place(p, reg, half.Entries, now, s.Name)
|
||||
}
|
||||
}
|
||||
if _, err := store.Update(key, p, rev); err != nil {
|
||||
note(fmt.Errorf("the held pipeline %s is not released: %w", key, err))
|
||||
continue
|
||||
}
|
||||
started++
|
||||
if err := s.Say(runnerclient.Subject(seat.EventWaitEnded, p.Module, p.Branch, commitOrGroup(p)), s.waitOf(p, held.Since, how)); err != nil {
|
||||
s.logf("wait-ended of %s is not said: %v", key, err)
|
||||
}
|
||||
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: p.Branch, Source: p.Source, Module: p.Module, Commit: p.Commit,
|
||||
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision}
|
||||
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, commitOrGroup(p)), ev); err != nil {
|
||||
s.logf("pipeline-started of %s is not said: %v", key, err)
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
|
||||
note(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return started, first
|
||||
}
|
||||
|
||||
// ReleaseExpired starts the pipelines held for the fingerprint whose bound key expired (§266, A-195): the 30 minutes passed
|
||||
// without the new runner running healthy, so the running runner runs them, and each started is said late through Late (A-206).
|
||||
func (s *Starter) ReleaseExpired(store Updater, key string, view func(branch string) (Half, bool)) (int, error) {
|
||||
pkeys, err := store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
fp := ""
|
||||
held := map[string]seat.Held{}
|
||||
for _, k := range pkeys {
|
||||
if p, _, err := store.Get(k); err == nil && p.Held != nil && p.Outcome == seat.PipelineOutcomeRunning && HeldKey(p.Held.Fingerprint) == key {
|
||||
fp, held[k] = p.Held.Fingerprint, *p.Held
|
||||
}
|
||||
}
|
||||
if fp == "" {
|
||||
return 0, nil
|
||||
}
|
||||
n, err := s.Release(store, fp, "released", view)
|
||||
for k, h := range held {
|
||||
if p, _, gerr := store.Get(k); gerr == nil && p.Held == nil && s.Late != nil {
|
||||
s.Late(p, h)
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// heldFingerprints are the fingerprints of the runner the held pipelines in store await, each once.
|
||||
func heldFingerprints(store Updater) []string {
|
||||
pkeys, err := store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var fps []string
|
||||
for _, key := range pkeys {
|
||||
if p, _, err := store.Get(key); err == nil && p.Held != nil && p.Outcome == seat.PipelineOutcomeRunning && !slices.Contains(fps, p.Held.Fingerprint) {
|
||||
fps = append(fps, p.Held.Fingerprint)
|
||||
}
|
||||
}
|
||||
return fps
|
||||
}
|
||||
|
||||
// Release starts by hand the pipelines held for the runner's new version with the running runner, recorded in the hand-act
|
||||
// log first (§276, A-206): each says wait-ended `released` and pipeline-started, and the bound of each wait is dropped. It
|
||||
// says how many it started.
|
||||
func (v Verbs) Release(_ string, in seat.ReleaseInput) (seat.ReleaseAnswer, error) {
|
||||
if v.Starter == nil || v.View == nil {
|
||||
return seat.ReleaseAnswer{}, errors.New("the runner cannot release a pipeline: it holds no starter or no catalog half")
|
||||
}
|
||||
if err := v.record(seat.VerbRelease, "started the pipelines held for the new runner by hand", in.Why); err != nil {
|
||||
return seat.ReleaseAnswer{}, err
|
||||
}
|
||||
fps := heldFingerprints(v.Hearer.Store)
|
||||
n, err := v.Starter.Release(v.Hearer.Store, "", "released", v.View)
|
||||
if n > 0 && v.Drop != nil {
|
||||
for _, fp := range fps {
|
||||
if derr := v.Drop(HeldKey(fp)); derr != nil && err == nil {
|
||||
err = fmt.Errorf("the bound of the wait for %s is not dropped: %w", fp, derr)
|
||||
}
|
||||
}
|
||||
}
|
||||
return seat.ReleaseAnswer{Released: int64(n)}, err
|
||||
}
|
||||
|
||||
// HeldTTL is how long the bound of a wait that counts from the build end at from still has to run at now: the 30 minutes less what passed since
|
||||
// the end, at least a second, the least a key's time to live holds (design 47 G §266, A-206).
|
||||
func HeldTTL(from, now time.Time) time.Duration {
|
||||
return max(HeldBound-now.Sub(from), time.Second)
|
||||
}
|
||||
@@ -1,245 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// memCreated is memStore with the Create a start writes through.
|
||||
type memCreated struct{ memStore }
|
||||
|
||||
func (m memCreated) Create(key string, value []byte) error {
|
||||
if _, ok := m.memStore[key]; ok {
|
||||
return ErrExists
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(value, &p); err != nil {
|
||||
return err
|
||||
}
|
||||
m.memStore[key] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{p, 1}
|
||||
return nil
|
||||
}
|
||||
|
||||
// A take-in giving the runner's own module a new fingerprint holds every pipeline it starts, a later take-in's too while one is
|
||||
// held, with no step asked; the new runner running healthy releases them, placing and asking their steps (design 47 G §65, §266,
|
||||
// §276, §534; A-195).
|
||||
func TestPipelinesAreHeldForTheNewRunnerAndReleasedWhenItRuns(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
entries["runner"] = catalog.CatalogEntry{Module: "runner"}
|
||||
store := memCreated{memStore{}}
|
||||
asks := &stepAsks{}
|
||||
var said []string
|
||||
var bounds []string
|
||||
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Self: "runner", SelfFp: "src2:old",
|
||||
Awaiting: Awaiting(store.memStore), Ends: func(string, string) (time.Time, bool) { return time.Unix(9, 0), true },
|
||||
Hold: func(fp string, _ time.Time) error { bounds = append(bounds, fp); return nil },
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
|
||||
Say: func(subject string, _ any) error {
|
||||
said = append(said, subject[strings.Index(subject, ".event.")+7:])
|
||||
return nil
|
||||
}}
|
||||
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
|
||||
n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{
|
||||
{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "runner", Commit: "c2", Fingerprint: "src2:new"}}}, view)
|
||||
if err != nil || n != 2 {
|
||||
t.Fatalf("take-in = %d, %v", n, err)
|
||||
}
|
||||
p, _, _ := store.Get("main.4.app")
|
||||
if p.Held == nil || p.Held.Fingerprint != "src2:new" || len(p.Steps) != 0 || len(asks.asks) != 0 {
|
||||
t.Fatalf("held = %+v, steps %d, asks %d", p.Held, len(p.Steps), len(asks.asks))
|
||||
}
|
||||
// The runner's own pipeline is never held: it runs on the running runner, so its pass lets the new runner be wanted (A-328).
|
||||
if p, _, _ := store.Get("main.4.runner"); p.Held != nil || p.Outcome != seat.PipelineOutcomeRunning {
|
||||
t.Errorf("the runner's own pipeline is held %+v, want it running on the running runner", p.Held)
|
||||
}
|
||||
if strings.Join(said, "|") != "wait-held.app.main.c1|pipeline-started.runner.main.c2" || len(bounds) != 1 || bounds[0] != "src2:new" {
|
||||
t.Errorf("said %v, bounds %v", said, bounds)
|
||||
}
|
||||
// A later take-in of the runner's own module while one is held is not held either (A-328).
|
||||
if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.3", Branch: "main", Source: 6, Entries: []catalog.TakeInEntry{{Module: "runner", Commit: "c4", Fingerprint: "src2:new"}}}, view); err != nil || n != 1 {
|
||||
t.Fatalf("a take-in of the runner alone = %d, %v", n, err)
|
||||
}
|
||||
if p, _, _ := store.Get("main.6.runner"); p.Held != nil {
|
||||
t.Errorf("the runner's own later pipeline is held %+v", p.Held)
|
||||
}
|
||||
// A later take-in not naming the runner is held for the same version.
|
||||
if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.2", Branch: "main", Source: 5, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c3", Fingerprint: "src2:bb"}}}, view); err != nil || n != 1 {
|
||||
t.Fatalf("second take-in = %d, %v", n, err)
|
||||
}
|
||||
if p, _, _ := store.Get("main.5.app"); p.Held == nil || len(asks.asks) != 0 {
|
||||
t.Fatalf("the later pipeline is not held: %+v", p.Held)
|
||||
}
|
||||
// Another fingerprint's release starts nothing.
|
||||
if n, err := s.Release(store.memStore, "src2:other", "ran", view); err != nil || n != 0 {
|
||||
t.Fatalf("release of another version = %d, %v", n, err)
|
||||
}
|
||||
said = nil
|
||||
n, err = s.Release(store.memStore, "src2:new", "ran", view)
|
||||
if err != nil || n != 2 {
|
||||
t.Fatalf("release = %d, %v", n, err)
|
||||
}
|
||||
p, _, _ = store.Get("main.4.app")
|
||||
if p.Held != nil || len(p.Steps) == 0 || len(asks.asks) == 0 {
|
||||
t.Errorf("after the release: held %+v, steps %d, asks %d", p.Held, len(p.Steps), len(asks.asks))
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "|"), "wait-ended.app.main.c1|pipeline-started.app.main.c1") {
|
||||
t.Errorf("said %v", said)
|
||||
}
|
||||
if _, held := Awaiting(store.memStore)(); held {
|
||||
t.Error("a pipeline is still held")
|
||||
}
|
||||
}
|
||||
|
||||
// The verb release starts the held pipelines by hand, recorded in the hand-act log first, saying why and ending each wait
|
||||
// `released`, and drops the bound of the wait; a release with no why does nothing (design 47 G §276, A-206).
|
||||
func TestReleaseVerbStartsTheHeldPipelinesByHand(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
entries["runner"] = catalog.CatalogEntry{Module: "runner"}
|
||||
store := memCreated{memStore{}}
|
||||
asks := &stepAsks{}
|
||||
var said []string
|
||||
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Self: "runner", SelfFp: "src2:old", Awaiting: Awaiting(store.memStore),
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
|
||||
Say: func(subject string, _ any) error {
|
||||
said = append(said, subject[strings.Index(subject, ".event.")+7:])
|
||||
return nil
|
||||
}}
|
||||
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
|
||||
if _, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{
|
||||
{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "runner", Commit: "c2", Fingerprint: "src2:new"}}}, view); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
acts, dropped := &actLog{}, []string{}
|
||||
v := Verbs{Hearer: &Hearer{Store: store.memStore}, Starter: s, View: view, Acts: acts,
|
||||
Drop: func(k string) error { dropped = append(dropped, k); return nil }}
|
||||
if _, err := v.Release("operator", seat.ReleaseInput{}); err == nil || len(*acts) != 0 {
|
||||
t.Fatalf("a release saying no why: %v, acts %v", err, *acts)
|
||||
}
|
||||
said = nil
|
||||
a, err := v.Release("operator", seat.ReleaseInput{Why: "the new runner will not come"})
|
||||
if err != nil || a.Released != 1 { // the runner's own pipeline was never held (A-328)
|
||||
t.Fatalf("release = %+v, %v", a, err)
|
||||
}
|
||||
if p, _, _ := store.Get("main.4.app"); p.Held != nil || len(p.Steps) == 0 || len(asks.asks) == 0 {
|
||||
t.Errorf("after the release: held %+v, steps %d", p.Held, len(p.Steps))
|
||||
}
|
||||
if len(*acts) != 1 || !strings.HasPrefix((*acts)[0], "mesh-pipeline-runner.release: ") || len(dropped) != 1 || dropped[0] != HeldKey("src2:new") {
|
||||
t.Errorf("acts %v, dropped %v", *acts, dropped)
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "|"), "wait-ended.app.main.c1") {
|
||||
t.Errorf("said %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound's expiry starts the held pipelines and says each as late through the starter's Late, with the awaited fingerprint and
|
||||
// since when; a pipeline that stays held (no view) is not said (design 47 G §266, A-206).
|
||||
func TestExpiredBoundSaysEachReleasedPipelineLate(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
store := memCreated{memStore{}}
|
||||
since := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
|
||||
for _, k := range []string{"main.4.app", "dev.4.app"} {
|
||||
store.memStore[k] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: k, Branch: strings.Split(k, ".")[0], Module: "app", Outcome: seat.PipelineOutcomeRunning,
|
||||
Held: &seat.Held{Since: since, Fingerprint: "src2:new"}}, 1}
|
||||
}
|
||||
var late []string
|
||||
s := &Starter{Pipelines: store, Steps: &stepAsks{}, Name: "the-runner", Say: func(string, any) error { return nil },
|
||||
Registration: func(string) (catalog.Registration, bool) { return reg, true },
|
||||
Late: func(p seat.Pipeline, h seat.Held) {
|
||||
late = append(late, p.Pipeline+"@"+h.Fingerprint+"@"+h.Since.Format(time.RFC3339))
|
||||
}}
|
||||
view := func(b string) (Half, bool) { return Half{Entries: entries}, b == "main" }
|
||||
if n, err := s.ReleaseExpired(store.memStore, HeldKey("src2:new"), view); err == nil || n != 1 {
|
||||
t.Fatalf("release = %d, %v (the pipeline of a branch with no view stays held, an error)", n, err)
|
||||
}
|
||||
if len(late) != 1 || late[0] != "main.4.app@src2:new@2026-11-11T12:00:00Z" {
|
||||
t.Errorf("late %v", late)
|
||||
}
|
||||
}
|
||||
|
||||
// A wait of the step pipeline waits on the runner version (design 47 G §534, A-206).
|
||||
func TestHeldWaitIsOnTheRunnerVersion(t *testing.T) {
|
||||
w := (&Starter{}).waitOf(seat.Pipeline{Pipeline: "main.4.app", Branch: "main", Module: "app"}, time.Unix(1, 0), "")
|
||||
if w.Step != seat.WaitStepPipeline || w.On != "runner version" {
|
||||
t.Errorf("wait %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// The wait of a held pipeline counts from the build end of the awaited runner: wait-held says it as `clock-from` and the bound is
|
||||
// armed from it; while that build has not ended nothing is armed, and hearing its end arms the bound and says wait-held again
|
||||
// with `clock-from`; a failed end starts no clock (design 47 G §266, §269, A-181, A-206).
|
||||
func TestHeldWaitCountsFromTheBuildEndOfTheAwaitedRunner(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
store := memCreated{memStore{}}
|
||||
ends := &BuildEnds{}
|
||||
var waits []seat.Wait
|
||||
var armed []string
|
||||
s := &Starter{Pipelines: store, Steps: &stepAsks{}, Name: "the-runner", Self: "runner", SelfFp: "src2:old", Awaiting: Awaiting(store.memStore),
|
||||
Ends: ends.At, Hold: func(fp string, from time.Time) error {
|
||||
armed = append(armed, fp+"@"+from.Format(time.RFC3339))
|
||||
return nil
|
||||
},
|
||||
Registration: func(string) (catalog.Registration, bool) { return reg, true },
|
||||
Say: func(subject string, body any) error {
|
||||
if w, ok := body.(seat.Wait); ok && strings.Contains(subject, ".wait-held.") {
|
||||
waits = append(waits, w)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
|
||||
taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{
|
||||
{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "runner", Commit: "c2", Fingerprint: "src2:new"}}}
|
||||
end := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
|
||||
// The runner's build has not ended: held, no clock, no bound.
|
||||
if _, err := s.TakenIn(taken, view); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(waits) != 1 || waits[0].ClockFrom != nil || len(armed) != 0 { // the runner's own pipeline is never held (A-328)
|
||||
t.Fatalf("waits %+v, armed %v", waits, armed)
|
||||
}
|
||||
// A failed end starts no clock; a done end does.
|
||||
if _, _, ok := ends.Heard([]byte(`{"module":"runner","fingerprint":"src2:new","outcome":"failed"}`), end); ok {
|
||||
t.Error("a failed end was kept")
|
||||
}
|
||||
if m, fp, ok := ends.Heard([]byte(`{"module":"runner","fingerprint":"src2:new","outcome":"done"}`), end); !ok || m != "runner" || fp != "src2:new" {
|
||||
t.Fatalf("heard %q %q %v", m, fp, ok)
|
||||
}
|
||||
waits = nil
|
||||
if n, err := s.Clocked(store.memStore, "runner", "src2:new"); err != nil || n != 1 {
|
||||
t.Fatalf("clocked = %d, %v", n, err)
|
||||
}
|
||||
if len(waits) != 1 || waits[0].ClockFrom == nil || !waits[0].ClockFrom.Equal(end) || len(armed) != 1 || armed[0] != "src2:new@2026-11-11T12:00:00Z" {
|
||||
t.Errorf("waits %+v, armed %v", waits, armed)
|
||||
}
|
||||
// A pipeline held after the end is known says the clock at once.
|
||||
waits = nil
|
||||
taken.Key, taken.Source = "m.2", 5
|
||||
if _, err := s.TakenIn(taken, view); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(waits) == 0 || waits[0].ClockFrom == nil || !waits[0].ClockFrom.Equal(end) {
|
||||
t.Errorf("waits %+v", waits)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound counts from the build end: the 30 minutes less what passed since, never under a second.
|
||||
func TestHeldTTLCountsFromTheBuildEnd(t *testing.T) {
|
||||
end := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
after, want time.Duration
|
||||
}{{0, 30 * time.Minute}, {10 * time.Minute, 20 * time.Minute}, {45 * time.Minute, time.Second}} {
|
||||
if got := HeldTTL(end, end.Add(c.after)); got != c.want {
|
||||
t.Errorf("after %s the bound has %s, want %s", c.after, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -107,25 +107,6 @@ func (r Reads) Stands() (seat.StandsAnswer, error) {
|
||||
return seat.StandsAnswer{Stands: out}, nil
|
||||
}
|
||||
|
||||
// Waits is the pipelines held for the runner's new version, each as the wait of step `pipeline` it says (design 47 G §266, §276,
|
||||
// §534, A-195): the acting part awaited by its fingerprint, since when, and the pipeline.
|
||||
func (r Reads) Waits() (seat.StandsAnswer, error) {
|
||||
all, err := r.pipelines()
|
||||
if err != nil {
|
||||
return seat.StandsAnswer{}, err
|
||||
}
|
||||
out := []seat.Wait{}
|
||||
for _, p := range all {
|
||||
if p.Held == nil || p.Outcome != seat.PipelineOutcomeRunning {
|
||||
continue
|
||||
}
|
||||
out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit, Group: p.Group, Source: p.Source,
|
||||
Revision: p.Revision, Fingerprint: p.Held.Fingerprint, Step: seat.WaitStepPipeline, On: seat.WaitOnBuildNode, Since: p.Held.Since, V: 1})
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) })
|
||||
return seat.StandsAnswer{Stands: out}, nil
|
||||
}
|
||||
|
||||
// WhyCarried is where a step's result was carried from (§25, §83): the pipeline it names.
|
||||
func (r Reads) WhyCarried(in seat.WhyCarriedInput) (seat.WhyCarriedAnswer, error) {
|
||||
p, ok, err := r.pipeline(in.Pipeline)
|
||||
|
||||
@@ -129,22 +129,6 @@ func TestReads(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// runner_waits lists the pipelines held for the runner's new version as waits of step `pipeline` (design 47 G §276, A-195).
|
||||
func TestWaitsListTheHeldPipelines(t *testing.T) {
|
||||
since := time.Date(2026, 11, 12, 6, 0, 0, 0, time.UTC)
|
||||
held := seat.Pipeline{Pipeline: "main.4.app", Module: "app", Branch: "main", Commit: "c1", Source: 4, Outcome: seat.PipelineOutcomeRunning,
|
||||
Held: &seat.Held{Since: since, Fingerprint: "src2:new"}}
|
||||
plain := seat.Pipeline{Pipeline: "main.3.app", Module: "app", Branch: "main", Outcome: seat.PipelineOutcomeRunning}
|
||||
pipes := memKept{}
|
||||
put(t, pipes, held.Pipeline, held)
|
||||
put(t, pipes, plain.Pipeline, plain)
|
||||
r := Reads{Pipelines: pipes, Kept: memKept{}}
|
||||
got, err := r.Waits()
|
||||
if err != nil || len(got.Stands) != 1 || got.Stands[0].Step != seat.WaitStepPipeline || got.Stands[0].Fingerprint != "src2:new" || !got.Stands[0].Since.Equal(since) {
|
||||
t.Fatalf("waits = %+v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// show carries each store's fill (pipelines, step-outputs, kept, step-log) and kept the kept store's, read through the bundle's state
|
||||
// call, never counted from the keys; a fill that cannot be read is an error, not an answer without it (design 47 G §366, §623, A-214).
|
||||
func TestShowAndKeptCarryTheStoresFill(t *testing.T) {
|
||||
|
||||
@@ -35,8 +35,6 @@ type Begin struct {
|
||||
Group string
|
||||
Repository string
|
||||
Revision int64
|
||||
// Await is the fingerprint of the runner's own module this pipeline is held for (§266, A-195); empty when it is not held.
|
||||
Await string
|
||||
// Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them.
|
||||
Entries map[string]catalog.CatalogEntry
|
||||
// Published is the package's publish that started the pipeline, nil for any other start (A-202).
|
||||
@@ -76,16 +74,6 @@ type Starter struct {
|
||||
Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error)
|
||||
// Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none.
|
||||
Arm Arm
|
||||
// Self is the runner's own module and SelfFp the fingerprint it runs (A-149); a take-in giving Self another fingerprint
|
||||
// holds its pipelines for the new runner (§65, §266, A-195). Awaiting is the fingerprint the held pipelines await, while
|
||||
// any is held; Hold arms the bound of that wait (HeldBound) counting from the time the awaited build ended. All nil: nothing is held.
|
||||
Self, SelfFp string
|
||||
Awaiting func() (string, bool)
|
||||
Hold func(fp string, from time.Time) error
|
||||
// Ends says when the build of a module at a fingerprint ended (BuildEnds.At): the clock of the wait's bound (A-181, A-206).
|
||||
Ends func(module, fp string) (time.Time, bool)
|
||||
// Late says a pipeline the bound's expiry started with the running runner (A-206): the finding pipeline-waits.
|
||||
Late func(p seat.Pipeline, held seat.Held)
|
||||
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
|
||||
Fills *Fills
|
||||
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
|
||||
@@ -124,19 +112,6 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
if err := s.Fills.refuseStart(key); err != nil {
|
||||
return false, err
|
||||
}
|
||||
// The runner's own pipeline is never held: it runs on the running runner, as a build module's running version builds its
|
||||
// successor (§270, A-328), so its pass lets the new runner be wanted (§29).
|
||||
if b.Module != "" && b.Module == s.Self {
|
||||
b.Await = ""
|
||||
}
|
||||
if b.Await == "" && s.Awaiting != nil && b.Module != s.Self {
|
||||
if fp, ok := s.Awaiting(); ok {
|
||||
b.Await = fp
|
||||
}
|
||||
}
|
||||
if b.Await != "" {
|
||||
return s.startHeld(b, key, now, b.Await)
|
||||
}
|
||||
p := newPipeline(b, key, now)
|
||||
var asks []Asked
|
||||
if b.unheld {
|
||||
@@ -289,7 +264,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
|
||||
note(fmt.Errorf("the pipeline %s is not read: %w", key, err))
|
||||
continue
|
||||
}
|
||||
if p.Outcome != seat.PipelineOutcomeRunning || p.Held != nil || p.Run != "" || len(p.Steps) > 0 {
|
||||
if p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" || len(p.Steps) > 0 {
|
||||
continue
|
||||
}
|
||||
r, ok := reg(p)
|
||||
@@ -449,17 +424,9 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
|
||||
half, held := view(ev.Branch)
|
||||
started := 0
|
||||
var first error
|
||||
// A take-in giving the runner's own module a new fingerprint holds every other pipeline it starts for that runner (§65,
|
||||
// A-195, A-328); the runner's own runs now (Start).
|
||||
await := ""
|
||||
for _, e := range ev.Entries {
|
||||
if s.changesSelf(e.Module, e.Fingerprint) {
|
||||
await = e.Fingerprint
|
||||
}
|
||||
}
|
||||
for _, e := range ev.Entries {
|
||||
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint,
|
||||
Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries, Await: await, unheld: !held})
|
||||
Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries, unheld: !held})
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
|
||||
@@ -91,6 +91,24 @@ func TestStarts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// memCreated is memStore with the Create a start writes through.
|
||||
type memCreated struct{ memStore }
|
||||
|
||||
func (m memCreated) Create(key string, value []byte) error {
|
||||
if _, ok := m.memStore[key]; ok {
|
||||
return ErrExists
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(value, &p); err != nil {
|
||||
return err
|
||||
}
|
||||
m.memStore[key] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{p, 1}
|
||||
return nil
|
||||
}
|
||||
|
||||
type failing struct{}
|
||||
|
||||
func (failing) Create(string, []byte) error { return errors.New("bus away") }
|
||||
@@ -216,4 +234,22 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
|
||||
if n != 1 || err != nil || len(p.Steps) == 0 {
|
||||
t.Errorf("the half of next placed %d (%v): %d steps, want the publish's pipeline placed", n, err, len(p.Steps))
|
||||
}
|
||||
|
||||
// A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running,
|
||||
// no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more.
|
||||
var old seat.Pipeline
|
||||
if err := json.Unmarshal([]byte(`{"pipeline":"main.940.app","branch":"main","module":"app","commit":"c9","source":940,"outcome":"running",`+
|
||||
`"held":{"since":"2026-11-15T18:05:00Z","fingerprint":"src2:new"}}`), &old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.memStore["main.940.app"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{old, 1}
|
||||
before = len(asks.asks)
|
||||
n, err = s.Waiting(store.memStore, "main", late)
|
||||
p, _, _ = store.Get("main.940.app")
|
||||
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == before {
|
||||
t.Errorf("the pipeline a runner before this one set aside: placed %d (%v), %d steps, %d asks more, want it started as a newly started one", n, err, len(p.Steps), len(asks.asks)-before)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,9 +76,9 @@ func TestManifest(t *testing.T) {
|
||||
{"sent-objects 256 MiB", objectMiB[catalogue.SentObjectsLocal] == 256},
|
||||
// §333, §54, §44, §53, §376, §24, §37, §4, §77: what the runner hears.
|
||||
{"consumes", equalSets(m.Consumes, []string{
|
||||
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "mesh-delivery-builder.build.ended.**",
|
||||
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
|
||||
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
|
||||
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "node-engine.running", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
|
||||
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
|
||||
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
|
||||
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})},
|
||||
@@ -107,7 +107,7 @@ func TestManifest(t *testing.T) {
|
||||
"seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})},
|
||||
{"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})},
|
||||
// RN25 (§539): the runner's tools carry the prefix runner_.
|
||||
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_waits", "runner_release", "runner_unlogged", "runner_unlogged_append"})},
|
||||
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})},
|
||||
// A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery,
|
||||
// never by a tool of its own; runner_health stays a tool.
|
||||
{"health is the unit", healthTool(raw) == "unit"},
|
||||
|
||||
@@ -18,8 +18,7 @@
|
||||
"rerun",
|
||||
"accept",
|
||||
"cancel",
|
||||
"ask-again",
|
||||
"release"
|
||||
"ask-again"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -27,8 +26,6 @@
|
||||
"runner_first_views",
|
||||
"runner_first_view_again",
|
||||
"runner_health",
|
||||
"runner_waits",
|
||||
"runner_release",
|
||||
"runner_unlogged",
|
||||
"runner_unlogged_append"
|
||||
],
|
||||
@@ -37,7 +34,6 @@
|
||||
"mesh-builds.package-published",
|
||||
"mesh-builds.toolchain-agreed",
|
||||
"mesh-builds.step-ended",
|
||||
"mesh-delivery-builder.build.ended.**",
|
||||
"operator-channel.decided",
|
||||
"node-build-go.step-ended",
|
||||
"node-build-node.step-ended",
|
||||
@@ -45,7 +41,6 @@
|
||||
"node-build-go.ask-taken",
|
||||
"node-build-node.ask-taken",
|
||||
"node-build-image.ask-taken",
|
||||
"node-engine.running",
|
||||
"mesh-wants.step-ended",
|
||||
"mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision",
|
||||
|
||||
Reference in New Issue
Block a user