pipeline runner: no pipeline is set aside for the runner's own new version

A take-in that changes the runner starts its pipelines as any take-in does; the
state bounds' held keys, the wait's `on` value `runner version`, the finding
pipeline-waits, the verb release (and the tools runner_release and runner_waits
that only served it), the Held field and the runner's build-end and running
hearing go. A pipeline a previous runner left set aside is placed and asked
at the runner's start as a newly started one.

Builds: design 47 G A-446 (replaces A-195, A-206, A-218, §266's hold)
This commit is contained in:
2026-11-16 10:34:12 +01:00
parent 57025ccb3b
commit 070a5dd35b
25 changed files with 55 additions and 1224 deletions
@@ -27,7 +27,6 @@ const (
VerbAccept = "accept"
VerbCancel = "cancel"
VerbAskAgain = "ask-again"
VerbRelease = "release"
)
const (
@@ -46,7 +45,7 @@ const (
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
func Verbs() []string {
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain, VerbRelease}
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain}
}
// Events is every event published on the seat's subject: a new copy each call.
@@ -360,14 +359,6 @@ const (
GroupEndedOutcomeCancelled = "cancelled"
)
// Held is a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195).
type Held struct {
// the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195)
Fingerprint string `json:"fingerprint"`
// since when the pipeline is held, RFC 3339
Since time.Time `json:"since"`
}
// KeptResult is a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8).
type KeptResult struct {
// sha256:<hex> of what the package's tests read
@@ -452,7 +443,6 @@ type Pipeline struct {
Fingerprint string `json:"fingerprint,omitempty"`
// the gathered group, when the pipeline is its
Group string `json:"group,omitempty"`
Held *Held `json:"held,omitempty"`
Module string `json:"module"`
Outcome string `json:"outcome"`
// yes when the module is over its test budget (design 47 G section 419)
@@ -731,7 +721,7 @@ type Wait struct {
// for its end, how it ended: answered, ran, released, superseded
How string `json:"how,omitempty"`
Module string `json:"module"`
// what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206)
// what the step waits on (design 47 *Health and going back*)
On string `json:"on"`
// the pipeline's key, <branch>.<revision>.<module>, a branch escaped as a key's name is
Pipeline string `json:"pipeline"`
@@ -751,10 +741,9 @@ type Wait struct {
// The values of Wait's on.
const (
WaitOnPerson = "person"
WaitOnBuildNode = "build node"
WaitOnRevisionView = "revision view"
WaitOnRunnerVersion = "runner version"
WaitOnPerson = "person"
WaitOnBuildNode = "build node"
WaitOnRevisionView = "revision view"
)
// The values of Wait's step.
@@ -991,18 +980,6 @@ type AskAgainAnswer struct {
Result StepResult `json:"result"`
}
// ReleaseInput is what release takes.
type ReleaseInput struct {
// why, written with the caller the runtime proves in the hand-act log
Why string `json:"why"`
}
// ReleaseAnswer is what release answers.
type ReleaseAnswer struct {
// how many held pipelines were started
Released int64 `json:"released"`
}
// PipelineStartedEvent is the body of the event pipeline-started.
type PipelineStartedEvent = PipelineStarted
@@ -501,35 +501,6 @@
]
},
"class": "act"
},
{
"name": "release",
"description": "The pipelines the runner holds for its new version started by hand with the running runner, whatever the new version's state; written in the hand-act log (design 47 G 276, A-206).",
"input": {
"type": "object",
"properties": {
"why": {
"type": "string",
"description": "why, written with the caller the runtime proves in the hand-act log"
}
},
"required": [
"why"
]
},
"answer": {
"type": "object",
"properties": {
"released": {
"type": "integer",
"description": "how many held pipelines were started"
}
},
"required": [
"released"
]
},
"class": "act"
}
],
"events": [
@@ -719,25 +690,6 @@
},
"description": "how many tests a step ran and carried, per level (ADR 0666 section 8)"
},
"Held": {
"type": "object",
"properties": {
"since": {
"type": "string",
"format": "date-time",
"description": "since when the pipeline is held, RFC 3339"
},
"fingerprint": {
"type": "string",
"description": "the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195)"
}
},
"required": [
"since",
"fingerprint"
],
"description": "a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195)"
},
"ToolchainUsed": {
"type": "object",
"properties": {
@@ -1074,9 +1026,6 @@
"type": "boolean",
"description": "yes when the module is over its test budget (design 47 G section 419)"
},
"held": {
"$ref": "#/defs/Held"
},
"widens": {
"type": "array",
"items": {
@@ -1537,10 +1486,9 @@
"enum": [
"person",
"build node",
"revision view",
"runner version"
"revision view"
],
"description": "what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206)"
"description": "what the step waits on (design 47 *Health and going back*)"
},
"who": {
"type": "string",
@@ -17,7 +17,6 @@ export declare const verbs: {
readonly accept: "accept";
readonly cancel: "cancel";
readonly askAgain: "ask-again";
readonly release: "release";
};
export declare const events: {
@@ -154,14 +153,6 @@ export interface GroupEnded {
"why"?: string;
}
/** a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195) */
export interface Held {
/** the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195) */
"fingerprint": string;
/** since when the pipeline is held, RFC 3339 */
"since": string;
}
/** a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8) */
export interface KeptResult {
/** sha256:<hex> of what the package's tests read */
@@ -211,7 +202,6 @@ export interface Pipeline {
"fingerprint"?: string;
/** the gathered group, when the pipeline is its */
"group"?: string;
"held"?: Held;
"module": string;
"outcome": "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
/** yes when the module is over its test budget (design 47 G section 419) */
@@ -414,8 +404,8 @@ export interface Wait {
/** for its end, how it ended: answered, ran, released, superseded */
"how"?: string;
"module": string;
/** what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206) */
"on": "person" | "build node" | "revision view" | "runner version";
/** what the step waits on (design 47 *Health and going back*) */
"on": "person" | "build node" | "revision view";
/** the pipeline's key, <branch>.<revision>.<module>, a branch escaped as a key's name is */
"pipeline": string;
/** the catalog graph revision of the branch the pipeline runs for */
@@ -606,18 +596,6 @@ export interface AskAgainAnswer {
"result": StepResult;
}
/** is what release takes. */
export interface ReleaseInput {
/** why, written with the caller the runtime proves in the hand-act log */
"why": string;
}
/** is what release answers. */
export interface ReleaseAnswer {
/** how many held pipelines were started */
"released": number;
}
/** is the body of the event pipeline-started. */
export type PipelineStartedEvent = PipelineStarted;
@@ -17,7 +17,6 @@ export const verbs = {
accept: "accept",
cancel: "cancel",
askAgain: "ask-again",
release: "release",
};
export const events = {
@@ -107,34 +107,6 @@ func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time) stdio.
Run: func(map[string]any) (any, error) { return pipeline.Health(pipelines, consumers, now()), nil }}
}
// WaitsTool is the runner's tool runner_waits: the pipelines held for its new version, with the fingerprint each awaits and since
// when (design 47 G §276, §539, A-195).
func WaitsTool(r pipeline.Reads) stdio.Tool {
return stdio.Tool{Name: Prefix + "_waits",
Description: "The pipelines the pipeline runner holds until its new version runs healthy: each with its pipeline, the " +
"fingerprint of the runner it awaits and since when.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return r.Waits() }}
}
// ReleaseTool is the runner's tool runner_release: it starts the pipelines held for the runner's new version with the running
// runner, through the seat's verb release, which the process that holds the pipelines answers and records in the hand-act log
// first (design 47 G §276, A-206).
func ReleaseTool(ask func(key string, body any) (json.RawMessage, error)) stdio.Tool {
return stdio.Tool{Name: Prefix + "_release",
Description: "Start now, with the running pipeline runner, the pipelines it holds until its new version runs healthy " +
"(runner_waits lists them), each wait ending released. Written in the hand-act log (mesh-hand-acts).",
Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{
"why": map[string]any{"type": "string", "description": "why, in plain words"}}},
Run: func(args map[string]any) (any, error) {
in, err := as[seat.ReleaseInput](args)
if err != nil {
return nil, err
}
return ask("seat:"+seat.Seat+"."+seat.VerbRelease, in)
}}
}
// StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states
// pipelines and kept by `mesh/state.info`, step-outputs by the object store's fill and step-log by the log's; the runner's
// process writes nothing for it. A store whose fill cannot be read is an error naming it.
@@ -177,7 +149,6 @@ func main() {
func(n string) (*stdio.LogInfo, error) { return stdio.Log(n).Info() }, time.Now)
r := pipeline.Reads{Pipelines: kept{stdio.State(seat.StatePipelines)}, Kept: kept{stdio.State(seat.StateKept)}, Fill: fill}
tools := append(Tools(r), FirstViewTools(os.Getenv(stdio.ServedAs), stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
tools = append(tools, WaitsTool(r), ReleaseTool(stdio.Ask))
tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask},
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...)
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
@@ -4,7 +4,6 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"strings"
"testing"
@@ -34,21 +33,11 @@ func TestTools(t *testing.T) {
}
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
names = append(names, HealthTool(none{}, none{}, nil).Name)
// A-206: the tool runner_release asks the seat's verb release with its why.
var asked string
rel := ReleaseTool(func(key string, body any) (json.RawMessage, error) {
asked = key + fmt.Sprintf(" %+v", body)
return json.RawMessage(`{"released":1}`), nil
})
if _, err := rel.Run(map[string]any{"why": "w"}); err != nil || asked != "seat:mesh-pipeline-runner.release {Why:w}" {
t.Errorf("runner_release asked %q, %v", asked, err)
}
names = append(names, rel.Name)
// §606, §549: the unlogged entries are listed and appended by hand.
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
names = append(names, tl.Name)
}
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_health runner_release runner_unlogged runner_unlogged_append"
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept runner_first_views runner_first_view_again runner_health runner_unlogged runner_unlogged_append"
if got := strings.Join(names, " "); got != want {
t.Errorf("tools %q, want %q", got, want)
}
@@ -1,44 +0,0 @@
package main
import (
"context"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// hearBuildEnds hears the builder's stream of build ends, kept last per subject and read whole at the start, then followed
// (design 47 G A-181, §556): when the build of the runner's new version ended is the time the wait's bound counts from, and
// each end heard for a runner held pipelines await starts their clock (§266, A-206). A stream it cannot hear is said, and the
// bound of such a wait is not armed until the runner starts again.
func hearBuildEnds(ctx context.Context, cred process.Account, js jetstream.JetStream, ends *pipeline.BuildEnds, st *pipeline.Starter, store pipeline.Updater,
logf func(string, ...any)) {
stream := catalogue.StreamBusName(builds.BuildsStream)
cc, err := cred.HearStream(ctx, js, stream, func(m jetstream.Msg) {
defer m.Ack()
md, err := m.Metadata()
if err != nil {
logf("%s has no time the stream stamped: %v", m.Subject(), err)
return
}
module, fp, ok := ends.Heard(m.Data(), md.Timestamp)
if !ok {
return
}
if n, err := st.Clocked(store, module, fp); err != nil {
logf("the clock of the %d pipelines held for %s: %v", n, fp, err)
} else if n > 0 {
logf("the build of %s ended: the bound of the %d pipelines held for it counts from it", fp, n)
}
}, logf)
if err != nil {
logf("the stream %s is not heard, so no wait for the runner's new version has its bound armed until the runner starts again: %v", stream, err)
return
}
go func() { <-ctx.Done(); cc.Stop() }()
}
@@ -1,76 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/testbus"
"mesh-delivery-pipeline/internal/pipeline"
)
// The runner hears the builder's stream of build ends, kept last per subject, read whole at its start: the end of its new version's
// build, published before it started, starts the clock of the pipelines held for it, counted from the time the stream stamped the end
// (design 47 G A-181, §266, A-206).
func TestTheRunnerHearsTheBuildEndOfItsNewVersionAtItsStart(t *testing.T) {
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
stream, subject := catalogue.StreamBusName(builds.BuildsStream), process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}, MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
published := time.Now().UTC()
if _, err := js.Publish(ctx, process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEventName("mesh-delivery-pipeline", "src2:new")),
[]byte(`{"module":"mesh-delivery-pipeline","fingerprint":"src2:new","outcome":"done"}`)); err != nil {
t.Fatal(err)
}
cred := process.Account{Node: "n1", Module: "mesh-delivery-pipeline"}
durable := cred.SeatEventConsumerConfig([]string{subject}, true)
durable.DeliverPolicy = jetstream.DeliverNewPolicy
if _, err := js.CreateOrUpdateConsumer(ctx, stream, durable); err != nil {
t.Fatal(err)
}
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "pipelines", History: 1})
if err != nil {
t.Fatal(err)
}
store := pipelineStore{ctx: ctx, kv: kv}
held, _ := json.Marshal(seat.Pipeline{Pipeline: "main.4.app", Branch: "main", Module: "app", Outcome: seat.PipelineOutcomeRunning,
Held: &seat.Held{Since: published, Fingerprint: "src2:new"}})
if _, err := kv.Create(ctx, "main.4.app", held); err != nil {
t.Fatal(err)
}
ends := &pipeline.BuildEnds{}
armed := make(chan time.Time, 1)
st := &pipeline.Starter{Self: "mesh-delivery-pipeline", Ends: ends.At, Say: func(string, any) error { return nil },
Hold: func(fp string, from time.Time) error { armed <- from; return nil }}
hearBuildEnds(ctx, cred, js, ends, st, store, t.Logf)
select {
case from := <-armed:
if from.Before(published.Add(-time.Minute)) || from.After(time.Now().Add(time.Second)) {
t.Errorf("the bound counts from %s, not from the end published at %s", from, published)
}
case <-ctx.Done():
t.Fatal("the end of the new runner's build was not heard at the start")
}
}
@@ -2,9 +2,6 @@ package main
import (
"context"
"fmt"
"slices"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -12,34 +9,11 @@ import (
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// findingsWriter is the runner's finding writer as the mesh-sdk makes it.
type findingsWriter = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]
// KindPipelineWaits is the finding of pipelines held for the runner's new version that the wait's bound started with the running
// runner (design 47 G §266, A-206).
const KindPipelineWaits = "pipeline-waits"
// awaitedSaid opens the evidence entry of a pipeline-waits finding that names the awaited runner fingerprint, `awaited <fp>`
// (A-276): the runner clears a finding by that entry alone, never by its summary.
const awaitedSaid = "awaited "
// lateFinding is the finding pipeline-waits of one pipeline the bound's expiry started: scope module, id its key
// `<branch>.<id>.<module>`, naming the fingerprint of the new runner it waited for and since when, the fingerprint also in one
// evidence entry `awaited <fp>` (A-276).
func lateFinding(p seat.Pipeline, held seat.Held, runner, node string, at time.Time) condseat.Finding {
return condseat.Finding{Scope: views.FindingScope, ID: p.Pipeline, Kind: KindPipelineWaits, Severity: condseat.FindingSeverityWarning,
Evidence: []condseat.Evidence{{At: at, Said: awaitedSaid + held.Fingerprint}},
Headline: "the new pipeline runner did not run healthy in time",
Summary: fmt.Sprintf("the pipeline %s was held since %s for the runner at %s to run healthy; the bound passed and the running runner runs it",
p.Pipeline, held.Since.UTC().Format(time.RFC3339), held.Fingerprint),
Source: "bound of the wait for the new runner", Observer: runner, At: at,
Subject: condseat.Subject{Scope: views.FindingScope, ID: p.Module, Machine: node}}
}
// findingWriter is the runner's mesh-sdk held finding writer (design 47 G §461, §689, A-29): the runner raises its findings
// as they happen (a failed head, §33), so the writer holds what it has written and its statement says exactly that, with the
// reader's open findings. The writer
@@ -58,33 +32,3 @@ func findingWriter(ctx context.Context, cred process.Account, conn *nats.Conn, j
}
return w, nil
}
// heldFindings is what clearAwaited needs of the runner's held finding writer: what it holds and a clear.
type heldFindings interface {
Held() []condseat.Finding
Clear(ctx context.Context, scope, id, kind, why string) error
}
// clearAwaited clears exactly the pipeline-waits findings whose evidence names fp awaited (A-218, A-276): the node-engine seat's
// running says the runner at fp runs healthy. The writer holds what its state findings hold from its start, so a finding raised
// before a restart is cleared too. It says how many it cleared, and the first clear that failed.
func clearAwaited(ctx context.Context, w heldFindings, fp string) (int, error) {
if w == nil || fp == "" {
return 0, nil
}
n := 0
var first error
for _, f := range w.Held() {
if f.Kind != KindPipelineWaits || !slices.ContainsFunc(f.Evidence, func(e condseat.Evidence) bool { return e.Said == awaitedSaid+fp }) {
continue
}
if err := w.Clear(ctx, f.Scope, f.ID, f.Kind, "the runner at "+fp+" runs healthy"); err != nil {
if first == nil {
first = err
}
continue
}
n++
}
return n, first
}
@@ -1,110 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/testbus"
"mesh-delivery-pipeline/internal/pipeline"
)
// Design 47 G A-218, A-276: a pipeline-waits finding raised before the runner restarted is held again by the new process's writer,
// seeded from the state findings, and is cleared when the node-engine seat's running says the awaited runner fingerprint healthy;
// the finding awaiting another fingerprint stands.
func TestAPipelineWaitsFindingIsClearedAfterARestartWhenItsAwaitedRunnerRunsHealthy(t *testing.T) {
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
const module = "mesh-delivery-pipeline"
states := map[string]jetstream.KeyValue{}
for _, state := range []string{"findings", "pipelines", "bounds"} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1})
if err != nil {
t.Fatal(err)
}
states[state] = kv
}
// Where the writer says its findings, as the bus keeps the module's events.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.>"}}); err != nil {
t.Fatal(err)
}
cred := process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"}
conn, err := process.Connect(cred)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
since := time.Now().UTC().Add(-time.Hour)
before, err := findingWriter(ctx, cred, nc, js, conn.State("findings"), t.Logf)
if err != nil {
t.Fatal(err)
}
for _, p := range []struct{ key, module, fp string }{{"main.4.app", "app", "src2:new"}, {"main.5.web", "web", "src2:newer"}} {
if err := before.Write(ctx, lateFinding(seat.Pipeline{Pipeline: p.key, Module: p.module, Branch: "main"},
seat.Held{Since: since, Fingerprint: p.fp}, module, "n1", since.Add(30*time.Minute))); err != nil {
t.Fatal(err)
}
}
before.Close()
keys := func() map[string]bool {
out := map[string]bool{}
ks, err := states["findings"].Keys(ctx)
if err != nil && err != jetstream.ErrNoKeysFound {
t.Fatal(err)
}
for _, k := range ks {
out[k] = true
}
return out
}
app, web := "module.main.4.app.pipeline-waits", "module.main.5.web.pipeline-waits"
if k := keys(); !k[app] || !k[web] {
t.Fatalf("the state findings holds %v before the restart, want %s and %s", k, app, web)
}
// The restart: a new process's writer holds what its state findings hold.
after, err := findingWriter(ctx, cred, nc, js, conn.State("findings"), t.Logf)
if err != nil {
t.Fatal(err)
}
defer after.Close()
if len(after.Held()) != 2 {
t.Fatalf("the writer after the restart holds %+v, want both findings", after.Held())
}
hub := &oneHub{}
starter := &pipeline.Starter{Logf: t.Logf}
if err := hearRunning(ctx, hub, module, starter, pipelineStore{ctx: ctx, kv: states["pipelines"]}, states["bounds"], after,
func(string) (pipeline.Half, bool) { return pipeline.Half{}, false }, t.Logf); err != nil {
t.Fatal(err)
}
healthy := true
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: "n1", Module: module, Fingerprint: "src2:new", Healthy: &healthy, At: time.Now().UTC()})
var acks []string
hub.handle(dataMsg{data: raw, acks: &acks})
if k := keys(); k[app] || !k[web] {
t.Errorf("after the runner at src2:new runs healthy the state findings holds %v: want %s cleared and %s standing", k, app, web)
}
if held := after.Held(); len(held) != 1 || held[0].ID != "main.5.web" {
t.Errorf("the writer holds %+v, want only the finding awaiting src2:newer", held)
}
}
@@ -1,57 +0,0 @@
package main
import (
"context"
"slices"
"strings"
"testing"
"time"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/views"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// The bound's expiry's finding is pipeline-waits, scope module, id the pipeline's key, naming the awaited fingerprint and since
// when (design 47 G §266, A-206); it passes the writer's own check of a finding.
func TestLateFindingIsPipelineWaitsOfTheModuleScope(t *testing.T) {
since := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
f := lateFinding(seat.Pipeline{Pipeline: "main.4.app", Module: "app", Branch: "main"}, seat.Held{Since: since, Fingerprint: "src2:new"},
"mesh-delivery-pipeline", "n1", since.Add(30*time.Minute))
if f.Kind != "pipeline-waits" || f.Scope != views.FindingScope || f.ID != "main.4.app" || f.Severity != condseat.FindingSeverityWarning {
t.Errorf("finding %+v", f)
}
if !strings.Contains(f.Summary, "src2:new") || !strings.Contains(f.Summary, "2026-11-11T12:00:00Z") {
t.Errorf("summary %q", f.Summary)
}
if err := views.CheckFinding(f); err != nil {
t.Error(err)
}
// A-276: the awaited fingerprint is in one evidence entry, `awaited <fp>`.
if len(f.Evidence) != 1 || f.Evidence[0].Said != "awaited src2:new" {
t.Errorf("evidence %+v, want one entry `awaited src2:new`", f.Evidence)
}
// A-218, A-276: a healthy running of the runner at a fingerprint clears exactly the pipeline-waits findings whose evidence
// names it awaited: not one awaiting another, not another kind naming it, never by the summary.
other := lateFinding(seat.Pipeline{Pipeline: "main.5.web", Module: "web"}, seat.Held{Since: since, Fingerprint: "src2:newer"}, "r", "n1", since)
other.Summary = f.Summary // a summary naming src2:new is never read
kind := f
kind.Kind, kind.ID = "step-failed", "main.4.app.build"
held := &memFindings{held: []condseat.Finding{f, other, kind}}
if n, err := clearAwaited(context.Background(), held, "src2:new"); err != nil || n != 1 || !slices.Equal(held.cleared, []string{"module.main.4.app.pipeline-waits"}) {
t.Errorf("cleared %d (%v): %v, want the one pipeline-waits awaiting src2:new", n, err, held.cleared)
}
}
// memFindings is a held finding writer's Held and Clear.
type memFindings struct {
held []condseat.Finding
cleared []string
}
func (m *memFindings) Held() []condseat.Finding { return m.held }
func (m *memFindings) Clear(_ context.Context, scope, id, kind, _ string) error {
m.cleared = append(m.cleared, scope+"."+id+"."+kind)
return nil
}
@@ -137,20 +137,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
// for its step.
ends := &pipeline.BuildEnds{}
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
// The runner's own module and fingerprint (A-149): a take-in giving it another holds the pipelines it starts (§65, A-195).
Self: cred.Module, SelfFp: getenv("MESH_FINGERPRINT"), Awaiting: pipeline.Awaiting(pipelineStore{ctx: ctx, kv: pipelines}),
Ends: ends.At, Hold: armHeld(ctx, bounds),
// The bound's expiry writes the finding pipeline-waits for each pipeline it started (§266, A-206).
Late: func(p seat.Pipeline, held seat.Held) {
module, node := rv.Findings.ModuleNode()
if err := rv.Findings.Write(ctx, lateFinding(p, held, module, node, time.Now().UTC())); err != nil {
logger.Printf("the finding pipeline-waits of %s is not written: %v", p.Pipeline, err)
}
},
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has}
placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error {
n, err := starter.PlaceRun(ev, half, rv.Catalog.View)
@@ -162,10 +151,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if !ok {
return
}
// Pipelines held for the fingerprint this runner runs are started now: it is the new runner (§19, A-195).
if n, err := starter.Release(pipelineStore{ctx: ctx, kv: pipelines}, starter.SelfFp, "ran", rv.Catalog.View); err != nil {
logger.Printf("the pipelines held for the runner at %s: %d released, then %v", starter.SelfFp, n, err)
}
// A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327).
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err)
@@ -234,23 +219,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil {
return err
}
store := pipelineStore{ctx: ctx, kv: pipelines}
expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
if err := watchBounds(ctx, bounds, func(key string) {
if !pipeline.IsHeldKey(key) {
expired(key)
return
}
if n, err := starter.ReleaseExpired(store, key, rv.Catalog.View); err != nil {
logger.Printf("the pipelines held for the runner, whose %s passed: %d released, then %v", key, n, err)
} else {
logger.Printf("the runner's new version did not run healthy within %s: %d held pipelines started with the running runner", pipeline.HeldBound, n)
}
}, logger.Printf); err != nil {
return err
}
hearBuildEnds(ctx, cred, js, ends, starter, store, logger.Printf)
if err := hearRunning(ctx, heard, cred.Module, starter, store, bounds, rv.Findings, rv.Catalog.View, logger.Printf); err != nil {
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
return err
}
if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil {
@@ -6,11 +6,9 @@ import (
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
@@ -178,61 +176,6 @@ func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm {
}
}
// armHeld is the bound of the pipelines held for the runner's new version: a key of the state bounds named for the awaited
// fingerprint, with its own time to live of 30 minutes counted from the end of that runner's build (design 47 G §266, A-195, A-77,
// A-206).
func armHeld(ctx context.Context, bounds jetstream.KeyValue) func(string, time.Time) error {
return func(fp string, from time.Time) error {
_, err := bounds.Create(ctx, pipeline.HeldKey(fp), []byte(`{"fingerprint":"`+fp+`"}`), jetstream.KeyTTL(pipeline.HeldTTL(from, time.Now())))
if errors.Is(err, jetstream.ErrKeyExists) {
return nil
}
return err
}
}
// hearRunning releases the pipelines held for the runner's new version when the node-engine seat says the runner's own module
// runs that fingerprint healthy, on any node (§266, §276, A-195); its bound is then deleted, and the pipeline-waits findings
// that await it are cleared (A-218, A-276). A running with no health reads as healthy (§520). Heard again when the release or
// a clear could not be written.
func hearRunning(ctx context.Context, hub handlers, self string, st *pipeline.Starter, store pipeline.Updater, bounds jetstream.KeyValue,
findings heldFindings, view func(string) (pipeline.Half, bool), logf func(string, ...any)) error {
return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) {
var ev nodeengine.Running
if err := json.Unmarshal(m.Data(), &ev); err != nil {
logf("%s is no running: %v", m.Subject(), err)
_ = m.Term()
return
}
if ev.Module != self || ev.Stopped || ev.Fingerprint == "" || (ev.Healthy != nil && !*ev.Healthy) {
_ = m.Ack()
return
}
n, err := st.Release(store, ev.Fingerprint, "ran", view)
if err != nil {
logf("the pipelines held for %s: %d released, then %v; heard again", ev.Fingerprint, n, err)
_ = m.Nak()
return
}
if n > 0 {
logf("the runner runs %s on %s: %d held pipelines released", ev.Fingerprint, ev.Node, n)
if err := bounds.Delete(ctx, pipeline.HeldKey(ev.Fingerprint)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
logf("the bound of the wait for %s is not deleted: %v", ev.Fingerprint, err)
}
}
cleared, err := clearAwaited(ctx, findings, ev.Fingerprint)
if cleared > 0 {
logf("the runner runs %s on %s: %d pipeline-waits findings cleared", ev.Fingerprint, ev.Node, cleared)
}
if err != nil {
logf("a pipeline-waits finding awaiting %s is not cleared: %v; heard again", ev.Fingerprint, err)
_ = m.Nak()
return
}
_ = m.Ack()
})
}
// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32,
// §535, A-77), and says a drop no holder answered unanswered (A-222): no timer in the process.
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
@@ -41,8 +41,7 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu
return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v))
}
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, ask-again and
// release (§64, §306; RN12, RN14; A-206).
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again.
func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }),
@@ -52,7 +51,6 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }),
seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }),
seat.VerbRelease: verbOf(func(c string, in seat.ReleaseInput) (any, error) { return v.Release(c, in) }),
}
}
@@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) {
got = append(got, v)
}
slices.Sort(got)
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbRelease, seat.VerbReview, seat.VerbStart, seat.VerbRun}
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun}
slices.Sort(want)
if !slices.Equal(got, want) {
t.Errorf("the process answers %v, want %v", got, want)
-2
View File
@@ -9,13 +9,11 @@ require (
)
require (
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git v0.0.0-00010101000000-000000000000
@@ -1,46 +0,0 @@
package pipeline
import (
"encoding/json"
"sync"
"time"
)
// BuildEnds are the done builds' ends heard from the builder's stream of build ends (event build.ended, kept last per subject
// and read whole at the start, design 47 G A-181): when each build, by module and fingerprint, ended, which is the time a held
// wait's bound counts from (§266, A-206).
type BuildEnds struct {
mu sync.Mutex
at map[string]time.Time
}
// buildEnd is the part of the body of build.ended the runner reads.
type buildEnd struct {
Module string `json:"module"`
Fingerprint string `json:"fingerprint"`
Outcome string `json:"outcome"`
}
// Heard keeps the end a message of the stream says, at the time the stream stamped it; a failed end, or a body naming no module
// and fingerprint, is kept for nothing. It says the module and fingerprint it kept.
func (e *BuildEnds) Heard(data []byte, at time.Time) (module, fp string, ok bool) {
var b buildEnd
if err := json.Unmarshal(data, &b); err != nil || b.Module == "" || b.Fingerprint == "" || b.Outcome == "failed" {
return "", "", false
}
e.mu.Lock()
defer e.mu.Unlock()
if e.at == nil {
e.at = map[string]time.Time{}
}
e.at[b.Module+"\x00"+b.Fingerprint] = at.UTC()
return b.Module, b.Fingerprint, true
}
// At is when the build of the module at the fingerprint ended, false when no end was heard.
func (e *BuildEnds) At(module, fp string) (time.Time, bool) {
e.mu.Lock()
defer e.mu.Unlock()
t, ok := e.at[module+"\x00"+fp]
return t, ok
}
@@ -1,269 +0,0 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"slices"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// HeldBound is how long the pipelines held for the runner's new version wait for it to run healthy (design 47 G §266: 30
// minutes for the runner's `pipeline` step, A-195): a key of the state `bounds` with its own time to live, never a timer.
const HeldBound = 30 * time.Minute
// HeldPrefix starts the key of a held wait's bound.
const HeldPrefix = "held-"
// HeldKey is the bound's key for the awaited fingerprint: a fingerprint holds characters a key's name does not.
func HeldKey(fp string) string {
sum := sha256.Sum256([]byte(fp))
return HeldPrefix + hex.EncodeToString(sum[:12])
}
// IsHeldKey says whether a key of the state `bounds` is a held wait's bound.
func IsHeldKey(key string) bool {
return len(key) > len(HeldPrefix) && key[:len(HeldPrefix)] == HeldPrefix
}
// Awaiting is the awaited fingerprint of the pipelines held in store, "" and false when none is held (design 47 G §266, A-195):
// a pipeline started while one is held is held for the same version, so it never runs on the old runner.
func Awaiting(store Updater) func() (string, bool) {
return func() (string, bool) {
pkeys, err := store.Keys("*.*.*")
if err != nil {
return "", false
}
for _, key := range pkeys {
if p, _, err := store.Get(key); err == nil && p.Held != nil && p.Outcome == seat.PipelineOutcomeRunning {
return p.Held.Fingerprint, true
}
}
return "", false
}
}
// changesSelf says whether a take-in's entry is the runner's own module at a fingerprint other than the one that runs: its
// pipelines are held for the new runner (design 47 G §65, §266, A-195).
func (s *Starter) changesSelf(module, fp string) bool {
return s.Self != "" && module == s.Self && fp != "" && fp != s.SelfFp
}
// startHeld writes b's pipeline held for the runner's version awaiting, its steps neither placed nor asked, and says
// wait-held with step `pipeline`; pipeline-started is said when it is released (§74, §266).
func (s *Starter) startHeld(b Begin, key string, now time.Time, awaiting string) (bool, error) {
p := newPipeline(b, key, now)
p.Held = &seat.Held{Since: now, Fingerprint: awaiting}
if err := s.create(key, p); errors.Is(err, ErrExists) {
s.logf("%s is started already: this start writes nothing", key)
return false, nil
} else if err != nil {
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
from := s.clockOf(awaiting)
if from != nil && s.Hold != nil {
if err := s.Hold(awaiting, *from); err != nil {
s.logf("the bound of the wait for the runner at %s is not armed: %v", awaiting, err)
}
}
w := s.waitOf(p, now, "")
w.ClockFrom = from
if err := s.Say(runnerclient.Subject(seat.EventWaitHeld, b.Module, b.Branch, commitOrGroup(p)), w); err != nil {
s.logf("wait-held of %s is not said: %v", key, err)
}
return true, nil
}
// clockOf is when the build of the runner at fp ended, the time the bound counts from, nil while it has not (§266, A-181, A-206).
func (s *Starter) clockOf(fp string) *time.Time {
if s.Ends == nil {
return nil
}
if at, ok := s.Ends(s.Self, fp); ok {
return &at
}
return nil
}
// Clocked starts the clock of the pipelines held for the runner at fp, the end of whose build was heard: it arms the bound from that
// end and says wait-held again for each with `clock-from`, so the watchman times it by the bound (§269, A-206). It says how many
// pipelines it said, and nothing when none is held for fp or the end is not known.
func (s *Starter) Clocked(store Updater, module, fp string) (int, error) {
if module != s.Self {
return 0, nil
}
from := s.clockOf(fp)
if from == nil {
return 0, nil
}
pkeys, err := store.Keys("*.*.*")
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
said := 0
var first error
for _, key := range pkeys {
p, _, err := store.Get(key)
if err != nil || p.Held == nil || p.Outcome != seat.PipelineOutcomeRunning || p.Held.Fingerprint != fp {
continue
}
if said == 0 && s.Hold != nil {
if err := s.Hold(fp, *from); err != nil {
first = fmt.Errorf("the bound of the wait for the runner at %s is not armed: %w", fp, err)
}
}
w := s.waitOf(p, p.Held.Since, "")
w.ClockFrom = from
if err := s.Say(runnerclient.Subject(seat.EventWaitHeld, p.Module, p.Branch, commitOrGroup(p)), w); err != nil && first == nil {
first = fmt.Errorf("wait-held of %s is not said: %w", key, err)
}
said++
}
return said, first
}
func commitOrGroup(p seat.Pipeline) string {
if p.Group != "" {
return p.Group
}
return p.Commit
}
func (s *Starter) waitOf(p seat.Pipeline, since time.Time, how string) seat.Wait {
return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint, Group: p.Group,
Revision: p.Revision, Source: p.Source, Step: seat.WaitStepPipeline, On: seat.WaitOnRunnerVersion, Since: since, How: how}
}
// Release starts the pipelines held for fingerprint fp ("" for every held one) as the new runner running healthy lets
// them, or the bound's expiry or a hand release (how: `ran` or `released`; §266, §276): each has its steps placed and asked and
// says wait-ended and pipeline-started. A pipeline whose branch has no view yet stays held, and the first error is returned
// so the event is heard again. It says how many it started.
func (s *Starter) Release(store Updater, fp, how string, view func(branch string) (Half, bool)) (int, error) {
pkeys, err := store.Keys("*.*.*")
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
}
started := 0
var first error
note := func(err error) {
if first == nil {
first = err
}
}
for _, key := range pkeys {
p, rev, err := store.Get(key)
if err != nil || p.Held == nil || p.Outcome != seat.PipelineOutcomeRunning || (fp != "" && p.Held.Fingerprint != fp) {
continue
}
half, ok := view(p.Branch)
if !ok {
note(fmt.Errorf("the catalog half of %s is not held yet: %s stays held", p.Branch, key))
continue
}
held := *p.Held
p.Held = nil
var asks []Asked
if s.Registration != nil {
if reg, ok := s.Registration(p.Module); ok {
p.Steps, asks = Place(p, reg, half.Entries, now, s.Name)
}
}
if _, err := store.Update(key, p, rev); err != nil {
note(fmt.Errorf("the held pipeline %s is not released: %w", key, err))
continue
}
started++
if err := s.Say(runnerclient.Subject(seat.EventWaitEnded, p.Module, p.Branch, commitOrGroup(p)), s.waitOf(p, held.Since, how)); err != nil {
s.logf("wait-ended of %s is not said: %v", key, err)
}
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: p.Branch, Source: p.Source, Module: p.Module, Commit: p.Commit,
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision}
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, commitOrGroup(p)), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
note(err)
}
}
}
return started, first
}
// ReleaseExpired starts the pipelines held for the fingerprint whose bound key expired (§266, A-195): the 30 minutes passed
// without the new runner running healthy, so the running runner runs them, and each started is said late through Late (A-206).
func (s *Starter) ReleaseExpired(store Updater, key string, view func(branch string) (Half, bool)) (int, error) {
pkeys, err := store.Keys("*.*.*")
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
fp := ""
held := map[string]seat.Held{}
for _, k := range pkeys {
if p, _, err := store.Get(k); err == nil && p.Held != nil && p.Outcome == seat.PipelineOutcomeRunning && HeldKey(p.Held.Fingerprint) == key {
fp, held[k] = p.Held.Fingerprint, *p.Held
}
}
if fp == "" {
return 0, nil
}
n, err := s.Release(store, fp, "released", view)
for k, h := range held {
if p, _, gerr := store.Get(k); gerr == nil && p.Held == nil && s.Late != nil {
s.Late(p, h)
}
}
return n, err
}
// heldFingerprints are the fingerprints of the runner the held pipelines in store await, each once.
func heldFingerprints(store Updater) []string {
pkeys, err := store.Keys("*.*.*")
if err != nil {
return nil
}
var fps []string
for _, key := range pkeys {
if p, _, err := store.Get(key); err == nil && p.Held != nil && p.Outcome == seat.PipelineOutcomeRunning && !slices.Contains(fps, p.Held.Fingerprint) {
fps = append(fps, p.Held.Fingerprint)
}
}
return fps
}
// Release starts by hand the pipelines held for the runner's new version with the running runner, recorded in the hand-act
// log first (§276, A-206): each says wait-ended `released` and pipeline-started, and the bound of each wait is dropped. It
// says how many it started.
func (v Verbs) Release(_ string, in seat.ReleaseInput) (seat.ReleaseAnswer, error) {
if v.Starter == nil || v.View == nil {
return seat.ReleaseAnswer{}, errors.New("the runner cannot release a pipeline: it holds no starter or no catalog half")
}
if err := v.record(seat.VerbRelease, "started the pipelines held for the new runner by hand", in.Why); err != nil {
return seat.ReleaseAnswer{}, err
}
fps := heldFingerprints(v.Hearer.Store)
n, err := v.Starter.Release(v.Hearer.Store, "", "released", v.View)
if n > 0 && v.Drop != nil {
for _, fp := range fps {
if derr := v.Drop(HeldKey(fp)); derr != nil && err == nil {
err = fmt.Errorf("the bound of the wait for %s is not dropped: %w", fp, derr)
}
}
}
return seat.ReleaseAnswer{Released: int64(n)}, err
}
// HeldTTL is how long the bound of a wait that counts from the build end at from still has to run at now: the 30 minutes less what passed since
// the end, at least a second, the least a key's time to live holds (design 47 G §266, A-206).
func HeldTTL(from, now time.Time) time.Duration {
return max(HeldBound-now.Sub(from), time.Second)
}
@@ -1,245 +0,0 @@
package pipeline
import (
"encoding/json"
"strings"
"testing"
"time"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// memCreated is memStore with the Create a start writes through.
type memCreated struct{ memStore }
func (m memCreated) Create(key string, value []byte) error {
if _, ok := m.memStore[key]; ok {
return ErrExists
}
var p seat.Pipeline
if err := json.Unmarshal(value, &p); err != nil {
return err
}
m.memStore[key] = struct {
p seat.Pipeline
rev uint64
}{p, 1}
return nil
}
// A take-in giving the runner's own module a new fingerprint holds every pipeline it starts, a later take-in's too while one is
// held, with no step asked; the new runner running healthy releases them, placing and asking their steps (design 47 G §65, §266,
// §276, §534; A-195).
func TestPipelinesAreHeldForTheNewRunnerAndReleasedWhenItRuns(t *testing.T) {
reg, entries := placeFixture()
entries["runner"] = catalog.CatalogEntry{Module: "runner"}
store := memCreated{memStore{}}
asks := &stepAsks{}
var said []string
var bounds []string
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Self: "runner", SelfFp: "src2:old",
Awaiting: Awaiting(store.memStore), Ends: func(string, string) (time.Time, bool) { return time.Unix(9, 0), true },
Hold: func(fp string, _ time.Time) error { bounds = append(bounds, fp); return nil },
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
Say: func(subject string, _ any) error {
said = append(said, subject[strings.Index(subject, ".event.")+7:])
return nil
}}
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{
{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "runner", Commit: "c2", Fingerprint: "src2:new"}}}, view)
if err != nil || n != 2 {
t.Fatalf("take-in = %d, %v", n, err)
}
p, _, _ := store.Get("main.4.app")
if p.Held == nil || p.Held.Fingerprint != "src2:new" || len(p.Steps) != 0 || len(asks.asks) != 0 {
t.Fatalf("held = %+v, steps %d, asks %d", p.Held, len(p.Steps), len(asks.asks))
}
// The runner's own pipeline is never held: it runs on the running runner, so its pass lets the new runner be wanted (A-328).
if p, _, _ := store.Get("main.4.runner"); p.Held != nil || p.Outcome != seat.PipelineOutcomeRunning {
t.Errorf("the runner's own pipeline is held %+v, want it running on the running runner", p.Held)
}
if strings.Join(said, "|") != "wait-held.app.main.c1|pipeline-started.runner.main.c2" || len(bounds) != 1 || bounds[0] != "src2:new" {
t.Errorf("said %v, bounds %v", said, bounds)
}
// A later take-in of the runner's own module while one is held is not held either (A-328).
if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.3", Branch: "main", Source: 6, Entries: []catalog.TakeInEntry{{Module: "runner", Commit: "c4", Fingerprint: "src2:new"}}}, view); err != nil || n != 1 {
t.Fatalf("a take-in of the runner alone = %d, %v", n, err)
}
if p, _, _ := store.Get("main.6.runner"); p.Held != nil {
t.Errorf("the runner's own later pipeline is held %+v", p.Held)
}
// A later take-in not naming the runner is held for the same version.
if n, err := s.TakenIn(catalog.TakeInTaken{Key: "m.2", Branch: "main", Source: 5, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c3", Fingerprint: "src2:bb"}}}, view); err != nil || n != 1 {
t.Fatalf("second take-in = %d, %v", n, err)
}
if p, _, _ := store.Get("main.5.app"); p.Held == nil || len(asks.asks) != 0 {
t.Fatalf("the later pipeline is not held: %+v", p.Held)
}
// Another fingerprint's release starts nothing.
if n, err := s.Release(store.memStore, "src2:other", "ran", view); err != nil || n != 0 {
t.Fatalf("release of another version = %d, %v", n, err)
}
said = nil
n, err = s.Release(store.memStore, "src2:new", "ran", view)
if err != nil || n != 2 {
t.Fatalf("release = %d, %v", n, err)
}
p, _, _ = store.Get("main.4.app")
if p.Held != nil || len(p.Steps) == 0 || len(asks.asks) == 0 {
t.Errorf("after the release: held %+v, steps %d, asks %d", p.Held, len(p.Steps), len(asks.asks))
}
if !strings.Contains(strings.Join(said, "|"), "wait-ended.app.main.c1|pipeline-started.app.main.c1") {
t.Errorf("said %v", said)
}
if _, held := Awaiting(store.memStore)(); held {
t.Error("a pipeline is still held")
}
}
// The verb release starts the held pipelines by hand, recorded in the hand-act log first, saying why and ending each wait
// `released`, and drops the bound of the wait; a release with no why does nothing (design 47 G §276, A-206).
func TestReleaseVerbStartsTheHeldPipelinesByHand(t *testing.T) {
reg, entries := placeFixture()
entries["runner"] = catalog.CatalogEntry{Module: "runner"}
store := memCreated{memStore{}}
asks := &stepAsks{}
var said []string
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Self: "runner", SelfFp: "src2:old", Awaiting: Awaiting(store.memStore),
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
Say: func(subject string, _ any) error {
said = append(said, subject[strings.Index(subject, ".event.")+7:])
return nil
}}
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
if _, err := s.TakenIn(catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{
{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "runner", Commit: "c2", Fingerprint: "src2:new"}}}, view); err != nil {
t.Fatal(err)
}
acts, dropped := &actLog{}, []string{}
v := Verbs{Hearer: &Hearer{Store: store.memStore}, Starter: s, View: view, Acts: acts,
Drop: func(k string) error { dropped = append(dropped, k); return nil }}
if _, err := v.Release("operator", seat.ReleaseInput{}); err == nil || len(*acts) != 0 {
t.Fatalf("a release saying no why: %v, acts %v", err, *acts)
}
said = nil
a, err := v.Release("operator", seat.ReleaseInput{Why: "the new runner will not come"})
if err != nil || a.Released != 1 { // the runner's own pipeline was never held (A-328)
t.Fatalf("release = %+v, %v", a, err)
}
if p, _, _ := store.Get("main.4.app"); p.Held != nil || len(p.Steps) == 0 || len(asks.asks) == 0 {
t.Errorf("after the release: held %+v, steps %d", p.Held, len(p.Steps))
}
if len(*acts) != 1 || !strings.HasPrefix((*acts)[0], "mesh-pipeline-runner.release: ") || len(dropped) != 1 || dropped[0] != HeldKey("src2:new") {
t.Errorf("acts %v, dropped %v", *acts, dropped)
}
if !strings.Contains(strings.Join(said, "|"), "wait-ended.app.main.c1") {
t.Errorf("said %v", said)
}
}
// The bound's expiry starts the held pipelines and says each as late through the starter's Late, with the awaited fingerprint and
// since when; a pipeline that stays held (no view) is not said (design 47 G §266, A-206).
func TestExpiredBoundSaysEachReleasedPipelineLate(t *testing.T) {
reg, entries := placeFixture()
store := memCreated{memStore{}}
since := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
for _, k := range []string{"main.4.app", "dev.4.app"} {
store.memStore[k] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: k, Branch: strings.Split(k, ".")[0], Module: "app", Outcome: seat.PipelineOutcomeRunning,
Held: &seat.Held{Since: since, Fingerprint: "src2:new"}}, 1}
}
var late []string
s := &Starter{Pipelines: store, Steps: &stepAsks{}, Name: "the-runner", Say: func(string, any) error { return nil },
Registration: func(string) (catalog.Registration, bool) { return reg, true },
Late: func(p seat.Pipeline, h seat.Held) {
late = append(late, p.Pipeline+"@"+h.Fingerprint+"@"+h.Since.Format(time.RFC3339))
}}
view := func(b string) (Half, bool) { return Half{Entries: entries}, b == "main" }
if n, err := s.ReleaseExpired(store.memStore, HeldKey("src2:new"), view); err == nil || n != 1 {
t.Fatalf("release = %d, %v (the pipeline of a branch with no view stays held, an error)", n, err)
}
if len(late) != 1 || late[0] != "main.4.app@src2:new@2026-11-11T12:00:00Z" {
t.Errorf("late %v", late)
}
}
// A wait of the step pipeline waits on the runner version (design 47 G §534, A-206).
func TestHeldWaitIsOnTheRunnerVersion(t *testing.T) {
w := (&Starter{}).waitOf(seat.Pipeline{Pipeline: "main.4.app", Branch: "main", Module: "app"}, time.Unix(1, 0), "")
if w.Step != seat.WaitStepPipeline || w.On != "runner version" {
t.Errorf("wait %+v", w)
}
}
// The wait of a held pipeline counts from the build end of the awaited runner: wait-held says it as `clock-from` and the bound is
// armed from it; while that build has not ended nothing is armed, and hearing its end arms the bound and says wait-held again
// with `clock-from`; a failed end starts no clock (design 47 G §266, §269, A-181, A-206).
func TestHeldWaitCountsFromTheBuildEndOfTheAwaitedRunner(t *testing.T) {
reg, entries := placeFixture()
store := memCreated{memStore{}}
ends := &BuildEnds{}
var waits []seat.Wait
var armed []string
s := &Starter{Pipelines: store, Steps: &stepAsks{}, Name: "the-runner", Self: "runner", SelfFp: "src2:old", Awaiting: Awaiting(store.memStore),
Ends: ends.At, Hold: func(fp string, from time.Time) error {
armed = append(armed, fp+"@"+from.Format(time.RFC3339))
return nil
},
Registration: func(string) (catalog.Registration, bool) { return reg, true },
Say: func(subject string, body any) error {
if w, ok := body.(seat.Wait); ok && strings.Contains(subject, ".wait-held.") {
waits = append(waits, w)
}
return nil
}}
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
taken := catalog.TakeInTaken{Key: "m.1", Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{
{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "runner", Commit: "c2", Fingerprint: "src2:new"}}}
end := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
// The runner's build has not ended: held, no clock, no bound.
if _, err := s.TakenIn(taken, view); err != nil {
t.Fatal(err)
}
if len(waits) != 1 || waits[0].ClockFrom != nil || len(armed) != 0 { // the runner's own pipeline is never held (A-328)
t.Fatalf("waits %+v, armed %v", waits, armed)
}
// A failed end starts no clock; a done end does.
if _, _, ok := ends.Heard([]byte(`{"module":"runner","fingerprint":"src2:new","outcome":"failed"}`), end); ok {
t.Error("a failed end was kept")
}
if m, fp, ok := ends.Heard([]byte(`{"module":"runner","fingerprint":"src2:new","outcome":"done"}`), end); !ok || m != "runner" || fp != "src2:new" {
t.Fatalf("heard %q %q %v", m, fp, ok)
}
waits = nil
if n, err := s.Clocked(store.memStore, "runner", "src2:new"); err != nil || n != 1 {
t.Fatalf("clocked = %d, %v", n, err)
}
if len(waits) != 1 || waits[0].ClockFrom == nil || !waits[0].ClockFrom.Equal(end) || len(armed) != 1 || armed[0] != "src2:new@2026-11-11T12:00:00Z" {
t.Errorf("waits %+v, armed %v", waits, armed)
}
// A pipeline held after the end is known says the clock at once.
waits = nil
taken.Key, taken.Source = "m.2", 5
if _, err := s.TakenIn(taken, view); err != nil {
t.Fatal(err)
}
if len(waits) == 0 || waits[0].ClockFrom == nil || !waits[0].ClockFrom.Equal(end) {
t.Errorf("waits %+v", waits)
}
}
// The bound counts from the build end: the 30 minutes less what passed since, never under a second.
func TestHeldTTLCountsFromTheBuildEnd(t *testing.T) {
end := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
for _, c := range []struct {
after, want time.Duration
}{{0, 30 * time.Minute}, {10 * time.Minute, 20 * time.Minute}, {45 * time.Minute, time.Second}} {
if got := HeldTTL(end, end.Add(c.after)); got != c.want {
t.Errorf("after %s the bound has %s, want %s", c.after, got, c.want)
}
}
}
@@ -107,25 +107,6 @@ func (r Reads) Stands() (seat.StandsAnswer, error) {
return seat.StandsAnswer{Stands: out}, nil
}
// Waits is the pipelines held for the runner's new version, each as the wait of step `pipeline` it says (design 47 G §266, §276,
// §534, A-195): the acting part awaited by its fingerprint, since when, and the pipeline.
func (r Reads) Waits() (seat.StandsAnswer, error) {
all, err := r.pipelines()
if err != nil {
return seat.StandsAnswer{}, err
}
out := []seat.Wait{}
for _, p := range all {
if p.Held == nil || p.Outcome != seat.PipelineOutcomeRunning {
continue
}
out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit, Group: p.Group, Source: p.Source,
Revision: p.Revision, Fingerprint: p.Held.Fingerprint, Step: seat.WaitStepPipeline, On: seat.WaitOnBuildNode, Since: p.Held.Since, V: 1})
}
sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) })
return seat.StandsAnswer{Stands: out}, nil
}
// WhyCarried is where a step's result was carried from (§25, §83): the pipeline it names.
func (r Reads) WhyCarried(in seat.WhyCarriedInput) (seat.WhyCarriedAnswer, error) {
p, ok, err := r.pipeline(in.Pipeline)
@@ -129,22 +129,6 @@ func TestReads(t *testing.T) {
}
}
// runner_waits lists the pipelines held for the runner's new version as waits of step `pipeline` (design 47 G §276, A-195).
func TestWaitsListTheHeldPipelines(t *testing.T) {
since := time.Date(2026, 11, 12, 6, 0, 0, 0, time.UTC)
held := seat.Pipeline{Pipeline: "main.4.app", Module: "app", Branch: "main", Commit: "c1", Source: 4, Outcome: seat.PipelineOutcomeRunning,
Held: &seat.Held{Since: since, Fingerprint: "src2:new"}}
plain := seat.Pipeline{Pipeline: "main.3.app", Module: "app", Branch: "main", Outcome: seat.PipelineOutcomeRunning}
pipes := memKept{}
put(t, pipes, held.Pipeline, held)
put(t, pipes, plain.Pipeline, plain)
r := Reads{Pipelines: pipes, Kept: memKept{}}
got, err := r.Waits()
if err != nil || len(got.Stands) != 1 || got.Stands[0].Step != seat.WaitStepPipeline || got.Stands[0].Fingerprint != "src2:new" || !got.Stands[0].Since.Equal(since) {
t.Fatalf("waits = %+v, %v", got, err)
}
}
// show carries each store's fill (pipelines, step-outputs, kept, step-log) and kept the kept store's, read through the bundle's state
// call, never counted from the keys; a fill that cannot be read is an error, not an answer without it (design 47 G §366, §623, A-214).
func TestShowAndKeptCarryTheStoresFill(t *testing.T) {
@@ -35,8 +35,6 @@ type Begin struct {
Group string
Repository string
Revision int64
// Await is the fingerprint of the runner's own module this pipeline is held for (§266, A-195); empty when it is not held.
Await string
// Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them.
Entries map[string]catalog.CatalogEntry
// Published is the package's publish that started the pipeline, nil for any other start (A-202).
@@ -76,16 +74,6 @@ type Starter struct {
Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error)
// Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none.
Arm Arm
// Self is the runner's own module and SelfFp the fingerprint it runs (A-149); a take-in giving Self another fingerprint
// holds its pipelines for the new runner (§65, §266, A-195). Awaiting is the fingerprint the held pipelines await, while
// any is held; Hold arms the bound of that wait (HeldBound) counting from the time the awaited build ended. All nil: nothing is held.
Self, SelfFp string
Awaiting func() (string, bool)
Hold func(fp string, from time.Time) error
// Ends says when the build of a module at a fingerprint ended (BuildEnds.At): the clock of the wait's bound (A-181, A-206).
Ends func(module, fp string) (time.Time, bool)
// Late says a pipeline the bound's expiry started with the running runner (A-206): the finding pipeline-waits.
Late func(p seat.Pipeline, held seat.Held)
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
Fills *Fills
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
@@ -124,19 +112,6 @@ func (s *Starter) Start(b Begin) (bool, error) {
if err := s.Fills.refuseStart(key); err != nil {
return false, err
}
// The runner's own pipeline is never held: it runs on the running runner, as a build module's running version builds its
// successor (§270, A-328), so its pass lets the new runner be wanted (§29).
if b.Module != "" && b.Module == s.Self {
b.Await = ""
}
if b.Await == "" && s.Awaiting != nil && b.Module != s.Self {
if fp, ok := s.Awaiting(); ok {
b.Await = fp
}
}
if b.Await != "" {
return s.startHeld(b, key, now, b.Await)
}
p := newPipeline(b, key, now)
var asks []Asked
if b.unheld {
@@ -289,7 +264,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
note(fmt.Errorf("the pipeline %s is not read: %w", key, err))
continue
}
if p.Outcome != seat.PipelineOutcomeRunning || p.Held != nil || p.Run != "" || len(p.Steps) > 0 {
if p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" || len(p.Steps) > 0 {
continue
}
r, ok := reg(p)
@@ -449,17 +424,9 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half
half, held := view(ev.Branch)
started := 0
var first error
// A take-in giving the runner's own module a new fingerprint holds every other pipeline it starts for that runner (§65,
// A-195, A-328); the runner's own runs now (Start).
await := ""
for _, e := range ev.Entries {
if s.changesSelf(e.Module, e.Fingerprint) {
await = e.Fingerprint
}
}
for _, e := range ev.Entries {
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint,
Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries, Await: await, unheld: !held})
Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries, unheld: !held})
if err != nil && first == nil {
first = err
}
@@ -91,6 +91,24 @@ func TestStarts(t *testing.T) {
}
}
// memCreated is memStore with the Create a start writes through.
type memCreated struct{ memStore }
func (m memCreated) Create(key string, value []byte) error {
if _, ok := m.memStore[key]; ok {
return ErrExists
}
var p seat.Pipeline
if err := json.Unmarshal(value, &p); err != nil {
return err
}
m.memStore[key] = struct {
p seat.Pipeline
rev uint64
}{p, 1}
return nil
}
type failing struct{}
func (failing) Create(string, []byte) error { return errors.New("bus away") }
@@ -216,4 +234,22 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
if n != 1 || err != nil || len(p.Steps) == 0 {
t.Errorf("the half of next placed %d (%v): %d steps, want the publish's pipeline placed", n, err, len(p.Steps))
}
// A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running,
// no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more.
var old seat.Pipeline
if err := json.Unmarshal([]byte(`{"pipeline":"main.940.app","branch":"main","module":"app","commit":"c9","source":940,"outcome":"running",`+
`"held":{"since":"2026-11-15T18:05:00Z","fingerprint":"src2:new"}}`), &old); err != nil {
t.Fatal(err)
}
store.memStore["main.940.app"] = struct {
p seat.Pipeline
rev uint64
}{old, 1}
before = len(asks.asks)
n, err = s.Waiting(store.memStore, "main", late)
p, _, _ = store.Get("main.940.app")
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == before {
t.Errorf("the pipeline a runner before this one set aside: placed %d (%v), %d steps, %d asks more, want it started as a newly started one", n, err, len(p.Steps), len(asks.asks)-before)
}
}
@@ -76,9 +76,9 @@ func TestManifest(t *testing.T) {
{"sent-objects 256 MiB", objectMiB[catalogue.SentObjectsLocal] == 256},
// §333, §54, §44, §53, §376, §24, §37, §4, §77: what the runner hears.
{"consumes", equalSets(m.Consumes, []string{
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "mesh-delivery-builder.build.ended.**",
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "node-engine.running", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})},
@@ -107,7 +107,7 @@ func TestManifest(t *testing.T) {
"seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})},
{"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})},
// RN25 (§539): the runner's tools carry the prefix runner_.
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_waits", "runner_release", "runner_unlogged", "runner_unlogged_append"})},
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})},
// A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery,
// never by a tool of its own; runner_health stays a tool.
{"health is the unit", healthTool(raw) == "unit"},
+1 -6
View File
@@ -18,8 +18,7 @@
"rerun",
"accept",
"cancel",
"ask-again",
"release"
"ask-again"
]
}
],
@@ -27,8 +26,6 @@
"runner_first_views",
"runner_first_view_again",
"runner_health",
"runner_waits",
"runner_release",
"runner_unlogged",
"runner_unlogged_append"
],
@@ -37,7 +34,6 @@
"mesh-builds.package-published",
"mesh-builds.toolchain-agreed",
"mesh-builds.step-ended",
"mesh-delivery-builder.build.ended.**",
"operator-channel.decided",
"node-build-go.step-ended",
"node-build-node.step-ended",
@@ -45,7 +41,6 @@
"node-build-go.ask-taken",
"node-build-node.ask-taken",
"node-build-image.ask-taken",
"node-engine.running",
"mesh-wants.step-ended",
"mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision",