Merge branch 'feat/47-G-recover-runner' into feat/47-G-join8

This commit is contained in:
2026-11-12 00:14:02 +01:00
29 changed files with 3118 additions and 76 deletions
@@ -391,6 +391,7 @@ const (
// The values of LogEntry's step.
const (
LogEntryStepBuild = "build"
LogEntryStepLint = "lint"
LogEntryStepUnit = "unit"
LogEntryStepIntegration = "integration"
@@ -521,6 +522,7 @@ const (
// The values of PipelineStartedStepsItem's step.
const (
PipelineStartedStepsItemStepBuild = "build"
PipelineStartedStepsItemStepLint = "lint"
PipelineStartedStepsItemStepUnit = "unit"
PipelineStartedStepsItemStepIntegration = "integration"
@@ -651,6 +653,7 @@ const (
// The values of StepResult's step.
const (
StepResultStepBuild = "build"
StepResultStepLint = "lint"
StepResultStepUnit = "unit"
StepResultStepIntegration = "integration"
@@ -715,6 +718,7 @@ const (
// The values of Wait's step.
const (
WaitStepBuild = "build"
WaitStepLint = "lint"
WaitStepUnit = "unit"
WaitStepIntegration = "integration"
@@ -817,6 +821,7 @@ type WhyCarriedInput struct {
// The values of WhyCarriedInput's step.
const (
WhyCarriedInputStepBuild = "build"
WhyCarriedInputStepLint = "lint"
WhyCarriedInputStepUnit = "unit"
WhyCarriedInputStepIntegration = "integration"
@@ -878,6 +883,7 @@ type RerunInput struct {
// The values of RerunInput's step.
const (
RerunInputStepBuild = "build"
RerunInputStepLint = "lint"
RerunInputStepUnit = "unit"
RerunInputStepIntegration = "integration"
@@ -904,6 +910,7 @@ type AcceptInput struct {
// The values of AcceptInput's step.
const (
AcceptInputStepBuild = "build"
AcceptInputStepLint = "lint"
AcceptInputStepUnit = "unit"
AcceptInputStepIntegration = "integration"
+7 -50
View File
@@ -16,10 +16,6 @@
{
"name": "run",
"description": "The run on request of one or more members, each a module at a named commit, through their pipelines; delivers nothing, replacing the checker's try (ADR 0666 section 18). Says run-started.",
"callers": [
"sessions",
"operator"
],
"input": {
"type": "object",
"properties": {
@@ -61,10 +57,6 @@
{
"name": "start",
"description": "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log.",
"callers": [
"operator",
"healers"
],
"input": {
"type": "object",
"properties": {
@@ -101,11 +93,6 @@
{
"name": "show",
"description": "One pipeline whole: its steps, its step log and its outputs, with the stores' fill. Reads only.",
"callers": [
"persons",
"sessions",
"healers"
],
"input": {
"type": "object",
"properties": {
@@ -153,11 +140,6 @@
{
"name": "list",
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill. Reads only.",
"callers": [
"persons",
"sessions",
"healers"
],
"input": {
"type": "object",
"properties": {
@@ -200,11 +182,6 @@
{
"name": "stands",
"description": "What stands still, and since when: each step not ended and each wait held. Reads only.",
"callers": [
"persons",
"sessions",
"healers"
],
"input": {
"type": "object",
"properties": {}
@@ -228,11 +205,6 @@
{
"name": "why-carried",
"description": "Where a step's result was carried from, with the kept results it rests on. Reads only.",
"callers": [
"persons",
"sessions",
"healers"
],
"input": {
"type": "object",
"properties": {
@@ -243,6 +215,7 @@
"step": {
"type": "string",
"enum": [
"build",
"lint",
"unit",
"integration",
@@ -279,11 +252,6 @@
{
"name": "kept",
"description": "The kept results, by digest, package or level, with the store's fill. Reads only.",
"callers": [
"persons",
"sessions",
"healers"
],
"input": {
"type": "object",
"properties": {
@@ -320,9 +288,6 @@
{
"name": "review",
"description": "A code review's result for a commit and fingerprint, taken from the reviewer's account and refused from the commit's author (ADR 0666 section 22).",
"callers": [
"sessions"
],
"input": {
"type": "object",
"properties": {
@@ -373,10 +338,6 @@
{
"name": "rerun",
"description": "One step again; its step-ended says it is a rerun and the caller the runtime proved. Written in the hand-act log.",
"callers": [
"operator",
"healers"
],
"input": {
"type": "object",
"properties": {
@@ -387,6 +348,7 @@
"step": {
"type": "string",
"enum": [
"build",
"lint",
"unit",
"integration",
@@ -428,9 +390,6 @@
{
"name": "accept",
"description": "A failed optional step accepted, with who and why, in the hand-act log.",
"callers": [
"operator"
],
"input": {
"type": "object",
"properties": {
@@ -441,6 +400,7 @@
"step": {
"type": "string",
"enum": [
"build",
"lint",
"unit",
"integration",
@@ -479,10 +439,6 @@
{
"name": "cancel",
"description": "A pipeline ended cancelled, its waiting asks dropped by each build seat's drop. Written in the hand-act log.",
"callers": [
"operator",
"healers"
],
"input": {
"type": "object",
"properties": {
@@ -516,9 +472,6 @@
{
"name": "ask-again",
"description": "A declined approval step asked again (ADR 0666 section 10). Written in the hand-act log.",
"callers": [
"operator"
],
"input": {
"type": "object",
"properties": {
@@ -821,6 +774,7 @@
"step": {
"type": "string",
"enum": [
"build",
"lint",
"unit",
"integration",
@@ -1206,6 +1160,7 @@
"step": {
"type": "string",
"enum": [
"build",
"lint",
"unit",
"integration",
@@ -1456,6 +1411,7 @@
"step": {
"type": "string",
"enum": [
"build",
"lint",
"unit",
"integration",
@@ -1516,6 +1472,7 @@
"step": {
"type": "string",
"enum": [
"build",
"lint",
"unit",
"integration",
+7 -7
View File
@@ -164,7 +164,7 @@ export interface LogEntry {
/** the output's name in step-outputs */
"output"?: string;
"pipeline": string;
"step": "lint" | "unit" | "integration" | "composition" | "review" | "approval";
"step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval";
"what": "started" | "ended";
}
@@ -251,7 +251,7 @@ export interface PipelineEnded {
export interface PipelineStartedStepsItem {
"artifact"?: string;
"status": "required" | "optional" | "bypass";
"step": "lint" | "unit" | "integration" | "composition" | "review" | "approval";
"step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval";
}
/** a pipeline's start */
@@ -342,7 +342,7 @@ export interface StepResult {
"started"?: string;
/** the step's status in the module's policy */
"status": "required" | "optional" | "bypass";
"step": "lint" | "unit" | "integration" | "composition" | "review" | "approval";
"step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval";
"tests"?: TestCounts;
/** what the outcome rests on: the part not heard, the seat with no holder, what a waiting step waits on */
"why"?: string;
@@ -389,7 +389,7 @@ export interface Wait {
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
"source"?: number;
/** a pipeline step a wait inside a pipeline is in; pipeline is the acting step of design 47 G 203, for a wait of the entries the runner holds for its new version (design 47 G 534) */
"step": "lint" | "unit" | "integration" | "composition" | "review" | "approval" | "pipeline";
"step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval" | "pipeline";
"v": number;
/** for a wait on a person, who: the operator, a reviewer */
"who"?: string;
@@ -474,7 +474,7 @@ export interface WhyCarriedInput {
"artifact"?: string;
/** the pipeline's key */
"pipeline": string;
"step": "lint" | "unit" | "integration" | "composition" | "review" | "approval";
"step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval";
}
/** is what why-carried answers. */
@@ -517,7 +517,7 @@ export interface RerunInput {
"artifact"?: string;
/** the pipeline's key */
"pipeline": string;
"step": "lint" | "unit" | "integration" | "composition" | "review" | "approval";
"step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval";
/** why, written with the caller the runtime proves in the hand-act log */
"why": string;
}
@@ -533,7 +533,7 @@ export interface AcceptInput {
"artifact"?: string;
/** the pipeline's key */
"pipeline": string;
"step": "lint" | "unit" | "integration" | "composition" | "review" | "approval";
"step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval";
/** why, written with the caller the runtime proves in the hand-act log */
"why": string;
}
@@ -0,0 +1,107 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// toolCaller is a kept call of a seat's verb on the bus (process.Conn.Call).
type toolCaller func(key string, body any) (json.RawMessage, error)
// routerBucket is the bucket of the router's asks: the state `asks` of the seat operator-channel (the contract's StateAsks) in
// the bucket of the seat's holder on the seat list, which the seat mesh-module-assigner's verb assignments answers with seats
// (design 47 G §700, A-74); no module's name is in the program. (The indexer's function, to move into mesh-sdk.)
func routerBucket(call toolCaller) (string, error) {
raw, err := call("seat:"+mma.Seat+"."+mma.VerbAssignments, mma.AssignmentsInput{Seats: mma.AssignmentsInputSeatsTrue})
if err != nil {
return "", fmt.Errorf("who holds the seat %s is not asked of the seat %s: %w", oc.Seat, mma.Seat, err)
}
var held struct {
Seats []mma.SeatHolders `json:"seats"`
}
if err := json.Unmarshal(raw, &held); err != nil {
return "", fmt.Errorf("the answer of %s.%s is not its seat's: %w", mma.Seat, mma.VerbAssignments, err)
}
for _, s := range held.Seats {
if s.Seat == oc.Seat && len(s.Holders) > 0 {
return s.Holders[0].Module + "_" + oc.StateAsks, nil
}
}
return "", fmt.Errorf("no module holds the seat %s on the seat list, so the router's record of an ask is not read", oc.Seat)
}
// routerRecordOf is Hearer.Where: the router's own record of an ask, read under the runner's module's name through
// operatorchannelclient, whose Record is a get of that key (`<module>.<id>` of the seat's state asks) on the process's own bus
// connection (design 47 G §700, A-89): never the decided event's body.
func routerRecordOf(module string, call toolCaller, record func(bucket, id string) (*stdio.StateEntry, error)) func(id string) (string, *oc.Warrant, error) {
client := &operatorchannelclient.Client{Module: module, Record: func(id string) (*stdio.StateEntry, error) {
bucket, err := routerBucket(call)
if err != nil {
return nil, err
}
return record(bucket, id)
}}
return client.Where
}
// busAsker publishes the runner's approval ask on the seat operator-channel's accept subject under the runner's own name, its
// duplicate id the ask's own, so an ask published twice is kept once (§37).
type busAsker struct {
conn *nats.Conn
cred process.Account
}
func (b busAsker) Ask(a oc.Ask) error {
data, err := json.Marshal(a)
if err != nil {
return err
}
msg := nats.NewMsg(oc.AcceptSubjectAsk(b.cred.Module))
msg.Header.Set("x-event-id", a.ID)
msg.Header.Set("x-source", b.cred.Module)
msg.Header.Set("x-node", b.cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Header.Set(nats.MsgIdHdr, a.ID)
msg.Data = data
return b.conn.PublishMsg(msg)
}
var _ pipeline.Asker = busAsker{}
// hearDecided wakes the runner on the operator-channel's decided for its own asks (§37, A-89): the warrant is read from the
// router's record, so an event the record does not back yet is heard again.
func hearDecided(hub *process.EventHub, h *pipeline.Hearer, module string, logf func(string, ...any)) error {
return hub.Handle(oc.SubjectDecided(module), func(m jetstream.Msg) {
var ev oc.Warrant
if err := json.Unmarshal(m.Data(), &ev); err != nil {
logf("%s holds what is no decided event, dropped: %v", m.Subject(), err)
_ = m.Term()
return
}
_, err := h.OnDecided(ev)
switch {
case errors.Is(err, pipeline.ErrRefused), errors.Is(err, pipeline.ErrNoAsk):
logf("the decided of ask %s takes nothing: %v", ev.Ask, err)
_ = m.Term()
case err != nil:
logf("the decided of ask %s is not taken yet, heard again: %v", ev.Ask, err)
_ = m.Nak()
default:
_ = m.Ack()
}
})
}
@@ -0,0 +1,41 @@
package main
import (
"context"
"encoding/json"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// busStepAsker puts a step's ask on a build seat's work queue at the seat's accept subject, under the runner's own name; its
// duplicate id is the ask's own, so an ask put twice is kept once (design 47 G §22, §24).
type busStepAsker struct {
ctx context.Context
js jetstream.JetStream
cred process.Account
}
func (b busStepAsker) AskStep(accept, id string, body any) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
msg := nats.NewMsg(accept)
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", b.cred.Module)
msg.Header.Set("x-node", b.cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Header.Set(nats.MsgIdHdr, id)
msg.Data = data
_, err = b.js.PublishMsg(b.ctx, msg)
return err
}
var _ pipeline.StepAsker = busStepAsker{}
@@ -11,6 +11,7 @@ import (
"log"
"os"
"os/signal"
"sync/atomic"
"syscall"
"github.com/nats-io/nats.go"
@@ -48,6 +49,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
kept, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StateKept), "keep a package's passing result")
if err != nil {
return err
}
// The bound of each ask a build seat's holder must take: a key with its own time to live, its expiry heard (§535, A-77).
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound an ask no holder takes")
if err != nil {
return err
}
// The toolchain of each language as heard agreed, for the steps' asks (§416, A-104).
toolchains, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "toolchains"), "name a step's toolchain")
if err != nil {
return err
}
// The registrations and each watched branch's catalog half, first views then events (§21, §195, §200): until a
// branch's view is held the runner acts on nothing new of it, and says so.
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
@@ -56,26 +71,77 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer calls.Close()
hub := process.NewEventHub(logger.Printf)
rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, logger.Printf)
// A branch's view holding its first half asks again the steps placed in its running pipelines that were never asked
// (§88); the starter is made after the views, so it is reached through a pointer set once it is.
var resumer atomic.Pointer[func(string)]
rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) {
if f := resumer.Load(); f != nil {
(*f)(branch)
}
}, logger.Printf)
if err != nil {
return err
}
// A pipeline starts by compare-and-set on its key's absence and says pipeline-started (§54, §103); a publish starts
// one per dependent behind (§53, §334).
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf,
Say: func(subject string, body any) error {
data, err := json.Marshal(body)
if err == nil {
_, err = js.Publish(ctx, subject, data)
}
return err
}}
say := func(subject string, body any) error {
data, err := json.Marshal(body)
if err == nil {
_, err = js.Publish(ctx, subject, data)
}
return err
}
hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf)
hearer.Asker = busAsker{conn: conn, cred: cred}
hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask)
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
// for its step.
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains)}
resume := func(branch string) {
half, ok := rv.Catalog.View(branch)
if !ok {
return
}
n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half)
if err != nil {
logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err)
return
}
logger.Printf("the steps placed on %s and not yet asked: %d asked again", branch, n)
}
resumer.Store(&resume)
if err := hearStepEnded(hub, hearer, logger.Printf); err != nil {
return err
}
if err := hearToolchain(ctx, hub, toolchains, logger.Printf); err != nil {
return err
}
if err := hearAskTaken(ctx, hub, bounds, logger.Printf); err != nil {
return err
}
if err := watchBounds(ctx, bounds, hearer, logger.Printf); err != nil {
return err
}
if err := hearDecided(hub, hearer, cred.Module, logger.Printf); err != nil {
return err
}
if err := hearPublished(ctx, hub, starter, calls, rv.Catalog, logger.Printf); err != nil {
return err
}
if err := hearTakenIn(hub, starter, rv.Catalog, logger.Printf); err != nil {
return err
}
if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil {
logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err)
}
// The seat's write verbs are answered here, by the process that holds the pipelines (A-107).
stopVerbs, err := answerVerbs(calls, writeVerbs(ctx, hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls, bounds))
if err != nil {
return err
}
defer stopVerbs()
go func() { <-ctx.Done(); hub.Stop() }()
logger.Printf("holding %s", seat.Seat)
<-ctx.Done()
@@ -0,0 +1,48 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
)
// Design 47 G §700, A-74: the router's record of an ask is read under the indexer's module's name from the state asks of the
// seat operator-channel, in the bucket of the seat's holder on the seat list, and carries the ask's warrant.
func TestTheRoutersRecordIsReadFromTheBucketOfTheSeatsHolder(t *testing.T) {
var asked []string
call := func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
return json.RawMessage(`{"seats":[{"seat":"node-packet-filter","scope":"node","node":"n1","holders":[{"module":"pf","node":"n1"}]},` +
`{"seat":"operator-channel","scope":"mesh","holders":[{"module":"ops-router","node":"n2"}]}],"holds":2}`), nil
}
var bucket, id string
record := func(b, i string) (*stdio.StateEntry, error) {
bucket, id = b, i
return &stdio.StateEntry{Key: "mesh-delivery-pipeline." + i, Value: json.RawMessage(`{"state":"chosen","warrant":{"ask":"a1","asker":"mesh-delivery-pipeline","choice":"approve"}}`)}, nil
}
state, w, err := routerRecordOf("mesh-delivery-pipeline", call, record)("a1")
if err != nil || state != "chosen" || w == nil || w.Ask != "a1" {
t.Fatalf("Where = %q, %+v, %v; want the state chosen and the warrant of a1", state, w, err)
}
if bucket != "ops-router_asks" || id != "a1" || len(asked) != 1 || asked[0] != "seat:mesh-module-assigner.assignments" {
t.Errorf("read %q key %q after asking %v; want the holder's bucket ops-router_asks, the id, one question of the seat mesh-module-assigner", bucket, id, asked)
}
// No holder on the seat list: nothing is read, and the warrant stays unapplied with the reason said.
none := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"seats":[],"holds":0}`), nil }
if _, _, err := routerRecordOf("mesh-delivery-pipeline", none, record)("a1"); err == nil || !strings.Contains(err.Error(), "no module holds the seat operator-channel") {
t.Errorf("with no holder Where answered %v; want the reason", err)
}
}
// The asker the router and the decided event name is the runner's own module (its bus account's), never a constant: a hearer
// that named another refused every decided of its own asks.
func TestTheHearerIsNamedByTheRunnersOwnModule(t *testing.T) {
h := newHearer(context.Background(), nil, process.Account{Module: "the-runner"}, nil, nil, nil, nil)
if h.Name != "the-runner" {
t.Errorf("the hearer's name is %q, want its account's module", h.Name)
}
}
@@ -0,0 +1,190 @@
//go:build integration
package main
import (
"bytes"
"context"
"encoding/json"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
type syncBuffer struct {
mu sync.Mutex
b bytes.Buffer
}
func (s *syncBuffer) Write(p []byte) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.b.Write(p)
}
func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
// The runner whole against the real bus (design 47 G §22, §44, §54, §70, §482, A-107): started as its process is, it takes a
// take-in.taken and places a pipeline of the registered module, asks its build of the seat mesh-builds, takes the step-ended
// of the build, and ends the placed review step when a session calls `review`, answered by the process with the caller the
// bus proved; the pipeline then ends passed.
func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) {
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
}
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
}
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
if err != nil {
t.Fatal(err)
}
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
asks, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy})
if err != nil {
t.Fatal(err)
}
// The seats the runner calls, answered as their holders do.
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional"}}}
answer := func(subject string, body any) {
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": body})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
}
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: []catalog.Registration{reg}})
answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{NoneYet: true})
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
dir := t.TempDir()
broker := filepath.Join(dir, "broker")
raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"})
if err := os.WriteFile(broker, raw, 0o600); err != nil {
t.Fatal(err)
}
logs := &syncBuffer{}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() {
done <- run(runCtx, func(k string) string {
if k == process.BrokerFileEnv {
return broker
}
return ""
}, logs)
}()
defer func() { stop(); <-done }()
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
if err != nil {
t.Fatal(err)
}
// A take-in.taken names one entry of the source revision; the runner places the pipeline of the registered module.
taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m.1", Kind: "taken", Branch: "main", Source: 3,
Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}})
const key = "main.3.app"
var p seat.Pipeline
waitFor(t, ctx, logs, func() bool {
// Said until the runner holds the registration, so the pipeline is placed with its policy's steps.
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil {
t.Fatal(err)
}
return readPipeline(ctx, pipelines, key, &p) && len(p.Steps) > 1
})
build := stepOf(p, seat.StepResultStepBuild)
if build.Outcome != seat.StepResultOutcomeRunning || build.Ask == "" {
t.Fatalf("the build step = %+v", build)
}
// The build is asked of the seat mesh-builds: its ask is on its work queue.
waitFor(t, ctx, logs, func() bool { info, err := asks.Info(ctx); return err == nil && info.State.Msgs >= 1 })
// A session's review, while the build still runs, is answered by the runner's process, the caller the one the subject names.
in, _ := json.Marshal(seat.ReviewInput{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Outcome: "passed", Said: "reads well"})
var reply struct {
Result seat.ReviewAnswer `json:"result"`
Error string `json:"error"`
}
waitFor(t, ctx, logs, func() bool {
m, err := nc.Request("mesh.seat.mesh-pipeline-runner.call.review.agent~dev", in, 2*time.Second)
return err == nil && json.Unmarshal(m.Data, &reply) == nil
})
if reply.Error != "" || reply.Result.Pipeline != key || reply.Result.Result.By != "agent.dev" || reply.Result.Result.Outcome != "passed" {
t.Fatalf("review answered %+v (%s)", reply.Result, reply.Error)
}
if !readPipeline(ctx, pipelines, key, &p) || stepOf(p, seat.StepResultStepReview).By != "agent.dev" || p.Outcome != seat.PipelineOutcomeRunning {
t.Fatalf("pipeline after the review = %+v\n%s", p, logs.String())
}
// The seat's holder says the build ended; the runner takes it from the node's subject.
ended, _ := json.Marshal(builds.StepEnded{ID: build.Ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed",
Entries: []builds.BuildEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}})
if _, err := js.Publish(ctx, "mesh.seat.mesh-builds.event.step-ended.n1", ended); err != nil {
t.Fatal(err)
}
waitFor(t, ctx, logs, func() bool {
return readPipeline(ctx, pipelines, key, &p) && stepOf(p, seat.StepResultStepBuild).Outcome == seat.StepResultOutcomePassed
})
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed })
}
func stepOf(p seat.Pipeline, step string) seat.StepResult {
for _, s := range p.Steps {
if s.Step == step {
return s
}
}
return seat.StepResult{}
}
func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *seat.Pipeline) bool {
e, err := kv.Get(ctx, key)
if err != nil {
return false
}
*p = seat.Pipeline{}
return json.Unmarshal(e.Value(), p) == nil
}
// waitFor asks cond until it holds or the test's context ends, the module's journal in the failure.
func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bool) {
t.Helper()
for ctx.Err() == nil {
if cond() {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("it did not come about in time; the runner said:\n%s", logs.String())
}
@@ -69,3 +69,24 @@ func hearPublished(ctx context.Context, hub *process.EventHub, starter *pipeline
}()
})
}
// hearTakenIn starts the pipelines of each take-in.taken heard (§44, §54): one per entry, the message acknowledged once
// they are written, and heard again when a write failed, so the keys already written start nothing twice.
func hearTakenIn(hub *process.EventHub, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) {
var ev catalog.TakeInTaken
if err := json.Unmarshal(m.Data(), &ev); err != nil {
logf("%s is no take-in.taken: %v", m.Subject(), err)
_ = m.Ack()
return
}
n, err := starter.TakenIn(ev, v.View)
if err != nil {
logf("the take-in %s of %s: %d pipelines started, then %v; heard again", ev.Key, ev.Branch, n, err)
_ = m.Nak()
return
}
logf("the take-in %s of %s started %d pipelines", ev.Key, ev.Branch, n)
_ = m.Ack()
})
}
@@ -0,0 +1,270 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/nats-io/nats.go/jetstream"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// pipelineStore is the state pipelines as the hearer reads and writes it: a write by compare-and-set on the revision read (§27).
type pipelineStore struct {
ctx context.Context
kv jetstream.KeyValue
}
func (s pipelineStore) Get(key string) (seat.Pipeline, uint64, error) {
e, err := s.kv.Get(s.ctx, key)
if err != nil {
return seat.Pipeline{}, 0, err
}
var p seat.Pipeline
if err := json.Unmarshal(e.Value(), &p); err != nil {
return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
}
return p, e.Revision(), nil
}
func (s pipelineStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) {
raw, err := json.Marshal(p)
if err != nil {
return 0, err
}
rev, err := s.kv.Update(s.ctx, key, raw, revision)
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
return 0, pipeline.ErrChanged
}
return rev, err
}
func (s pipelineStore) Keys(filter string) ([]string, error) {
all, err := s.kv.Keys(s.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
var out []string
want := strings.Split(filter, ".")
for _, k := range all {
got := strings.Split(k, ".")
if len(got) != len(want) {
continue
}
ok := true
for i := range want {
ok = ok && (want[i] == "*" || want[i] == got[i])
}
if ok {
out = append(out, k)
}
}
return out, err
}
// newHearer is the hearer over the runner's own states: pipelines, kept, the step log on its stream and step-outputs on its
// objects (§27). Each store is opened where it is used, so one the grants part has not made yet fails that write loudly.
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue,
say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer {
return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module,
Log: func(e seat.LogEntry) error {
return say(cred.EventSubject("step-log."+keys.EscapeName(e.Pipeline)), e)
},
Keep: func(key string, k seat.KeptResult) error {
raw, err := json.Marshal(k)
if err == nil {
_, err = kept.Put(ctx, key, raw)
}
return err
},
Output: func(key string, data []byte) error {
o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs))
if err != nil {
return fmt.Errorf("the objects %s are not open: %w", seat.StateStepOutputs, err)
}
_, err = o.PutBytes(ctx, key, data)
return err
}}
}
// hearStepEnded takes each build seat's step-ended into its pipeline (§24; ADR 0666 §21): refused and said, never taken,
// when it is not as the runner asked it, and heard again when a write failed.
func hearStepEnded(hub *process.EventHub, h *pipeline.Hearer, logf func(string, ...any)) error {
for _, bs := range pipeline.StepSeats {
bs := bs
err := hub.Handle(bs.Prefix+"*", func(m jetstream.Msg) {
decode := bs.Decode
if decode == nil {
decode = func(raw []byte) (pipeline.StepEnded, error) {
var ev pipeline.StepEnded
return ev, json.Unmarshal(raw, &ev)
}
}
ev, err := decode(m.Data())
if err != nil {
logf("%s is no step-ended: %v", m.Subject(), err)
_ = m.Term()
return
}
_, err = h.OnStepEnded(bs, m.Subject(), ev)
switch {
case errors.Is(err, pipeline.ErrRefused):
logf("%v", err)
_ = m.Term()
case err != nil:
logf("the step-ended of ask %s is not taken yet, heard again: %v", ev.ID, err)
_ = m.Nak()
default:
_ = m.Ack()
}
})
if err != nil {
return err
}
}
return nil
}
// hearAskTaken deletes the bound of each ask a build seat's holder took (§602, §647): the ask is answered, so its five minutes
// end with no finding.
func hearAskTaken(ctx context.Context, hub *process.EventHub, bounds jetstream.KeyValue, logf func(string, ...any)) error {
for _, ts := range pipeline.TakenSeats {
ts := ts
err := hub.Handle(ts.Subject, func(m jetstream.Msg) {
var ev nodebuildgo.AskTaken
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
logf("%s is no ask-taken: %v", m.Subject(), err)
_ = m.Term()
return
}
if err := bounds.Delete(ctx, ev.Ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
logf("the bound of ask %s, taken on %s, is not deleted, heard again: %v", ev.Ask, ev.Node, err)
_ = m.Nak()
return
}
_ = m.Ack()
})
if err != nil {
return err
}
}
return nil
}
// armBound is the bound of an ask: a key of the state bounds with its own time to live, named for the ask.
func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm {
return func(ask string) error {
_, err := bounds.Create(ctx, ask, []byte(`{"ask":"`+ask+`"}`), jetstream.KeyTTL(pipeline.AskBound))
if errors.Is(err, jetstream.ErrKeyExists) {
return nil
}
return err
}
}
// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32,
// §535, A-77): no timer in the process.
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, h *pipeline.Hearer, logf func(string, ...any)) error {
w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
return err
}
go func() {
for {
select {
case <-ctx.Done():
_ = w.Stop()
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsAskKey(e.Key()) {
continue
}
if err := h.NotTaken(e.Key(), pipeline.AskBound); err != nil {
logf("the step of ask %s, which no holder took, is not ended: %v", e.Key(), err)
}
}
}
}()
return nil
}
// keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a
// name or lies beneath it, in one pass over the bucket.
func keptReader(ctx context.Context, kept jetstream.KeyValue) func([]string) (map[string][]nodebuildgo.KeptResult, error) {
return func(pkgs []string) (map[string][]nodebuildgo.KeptResult, error) {
w, err := kept.WatchAll(ctx, jetstream.IgnoreDeletes())
if err != nil {
return nil, err
}
defer w.Stop()
out := map[string][]nodebuildgo.KeptResult{}
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case e, ok := <-w.Updates():
if !ok {
return out, nil
}
if e == nil { // the end of what the bucket holds now
return out, nil
}
var k nodebuildgo.KeptResult
if err := json.Unmarshal(e.Value(), &k); err != nil {
continue
}
for _, name := range pkgs {
if pipeline.InPackage(k.Package, name) {
out[name] = append(out[name], k)
}
}
}
}
}
}
// toolchainReader reads the toolchain of a language as the runner heard it agreed, from its state toolchains (A-104); nil
// when none was heard.
func toolchainReader(ctx context.Context, toolchains jetstream.KeyValue) func(string) (*nodebuildgo.ToolchainUsed, error) {
return func(language string) (*nodebuildgo.ToolchainUsed, error) {
e, err := toolchains.Get(ctx, keys.EscapeName(language))
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return pipeline.ToolchainOf(e.Value())
}
}
// hearToolchain keeps each toolchain-agreed heard as `<language>` in the state toolchains (A-104): the newest of a language.
func hearToolchain(ctx context.Context, hub *process.EventHub, toolchains jetstream.KeyValue, logf func(string, ...any)) error {
return hub.Handle(builds.SubjectToolchainAgreed, func(m jetstream.Msg) {
t, err := pipeline.ToolchainOf(m.Data())
if err != nil || t == nil {
logf("%s is no toolchain-agreed: %v", m.Subject(), err)
_ = m.Term()
return
}
if _, err := toolchains.Put(ctx, keys.EscapeName(t.Language), m.Data()); err != nil {
logf("the toolchain of %s is not kept, heard again: %v", t.Language, err)
_ = m.Nak()
return
}
_ = m.Ack()
})
}
@@ -0,0 +1,60 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// verbOf makes the handler of one verb from its function of the proved caller and the decoded input: the caller is the bus
// user the call's subject names, and a call naming none is refused (design 47 G §394, A-107).
func verbOf[In any](do func(caller string, in In) (any, error)) process.SeatHandler {
return func(caller string, body json.RawMessage) (any, error) {
if caller == "" {
return nil, errors.New("the call names no caller the bus proved: it is refused")
}
var in In
if len(body) > 0 {
if err := json.Unmarshal(body, &in); err != nil {
return nil, fmt.Errorf("the arguments do not fit the verb's input: %w", err)
}
}
return do(caller, in)
}
}
// answerVerbs answers the seat's write verbs `review`, `rerun`, `accept` and `cancel` over the bus, each writing the state
// pipelines by compare-and-set (design 47 G §22, §70, §482, A-107). The returned stop ends the answers.
func answerVerbs(conn *process.Conn, v pipeline.Verbs) (func(), error) {
return conn.AnswerSeat(seat.Seat, map[string]process.SeatHandler{
seat.VerbReview: verbOf(func(c string, in seat.ReviewInput) (any, error) { return v.Review(c, in) }),
seat.VerbRerun: verbOf(func(c string, in seat.RerunInput) (any, error) { return v.Rerun(c, in) }),
seat.VerbAccept: verbOf(func(c string, in seat.AcceptInput) (any, error) { return v.Accept(c, in) }),
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
})
}
// dropBound deletes the bound of an ask a cancelled pipeline no longer waits on.
func dropBound(ctx context.Context, bounds jetstream.KeyValue) func(string) error {
return func(ask string) error {
if err := bounds.Delete(ctx, ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
}
// writeVerbs are the verbs over the hearer and the starter, their hand acts recorded through the seat mesh-hand-acts.
func writeVerbs(ctx context.Context, h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool),
calls *process.Conn, bounds jetstream.KeyValue) pipeline.Verbs {
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Acts: meshhandactsclient.Recorder{Ask: calls.Call}, Drop: dropBound(ctx, bounds)}
}
@@ -0,0 +1,24 @@
package main
import (
"encoding/json"
"testing"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
)
// A write verb is answered only for a caller the bus proved, and its input is the verb's own (design 47 G §394, A-107).
func TestAVerbIsAnsweredOnlyForAProvedCallerAndItsOwnInput(t *testing.T) {
var got seat.CancelInput
h := verbOf(func(c string, in seat.CancelInput) (any, error) { got = in; return c, nil })
if _, err := h("", json.RawMessage(`{"pipeline":"p","why":"w"}`)); err == nil {
t.Error("a call naming no proved caller was answered")
}
if _, err := h("operator", json.RawMessage(`{"pipeline":`)); err == nil {
t.Error("a malformed input was answered")
}
out, err := h("operator", json.RawMessage(`{"pipeline":"p","why":"w"}`))
if err != nil || out != "operator" || got.Pipeline != "p" || got.Why != "w" {
t.Errorf("answer = %v, %v, input %+v", out, err, got)
}
}
@@ -25,7 +25,7 @@ type runnerViews struct {
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
// after, then start hub once.
func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State,
call meshcatalogclient.Call, logf func(string, ...any)) (*runnerViews, error) {
call meshcatalogclient.Call, onReady func(branch string), logf func(string, ...any)) (*runnerViews, error) {
findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf)
if err != nil {
return nil, err
@@ -37,7 +37,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
// A watched branch first named by a registration gets its view of the catalog half (CT10), adopted through the reader (design
// 47 G §532, A-26): the reader keeps its status, raises and clears its findings through the module's writer, hears the
// seat's catalog-revision on the module's consumer and asks its first view at once.
cv := &pipeline.Views{Logf: logf, Open: func(branch string) (pipeline.BranchView, error) {
cv := &pipeline.Views{Logf: logf, OnReady: onReady, Open: func(branch string) (pipeline.BranchView, error) {
c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, nil)
if err != nil {
return nil, err
@@ -59,6 +59,8 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it.
type catalogView struct{ c *meshcatalogclient.Catalog }
func (v catalogView) Ready() <-chan struct{} { return v.c.Ready() }
func (v catalogView) Hear(ctx context.Context, data []byte) error { return v.c.Hear(ctx, data) }
func (v catalogView) View() (pipeline.Half, bool) {
+6
View File
@@ -9,10 +9,16 @@ require (
)
require (
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
github.com/klauspost/compress v1.20.1 // indirect
@@ -0,0 +1,218 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The approval step is an authorising ask of tier approve, the seat operator-channel's accepted kind ask under the
// runner's own name, its about the module and the watched branch only, so a newer commit's ask replaces the older one's
// (design 47 G §26, §37; ADR 0666 §10). It is asked last, once every other required step passed.
const (
OptionApprove = "approve"
OptionDecline = "decline"
)
// About is what an approval ask is about: the module and the watched branch, each a key's name, joined by a dot.
func About(module, branch string) string {
return keys.EscapeName(module) + "." + keys.EscapeName(branch)
}
// approvalDue is the index of the approval step to ask now (§26): waiting, not bypassed, every other step ended and every
// other required step passed or carried; -1 when none is due.
func approvalDue(p seat.Pipeline) int {
at := -1
for i, s := range p.Steps {
if s.Step == seat.StepResultStepApproval {
if s.Status != seat.StepResultStatusBypass && s.Outcome == seat.StepResultOutcomeWaiting {
at = i
}
continue
}
if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning {
return -1
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
return -1
}
}
return at
}
// ApprovalAsk is the ask of a pipeline's approval step. Its id and its expiry follow from the pipeline and from when the
// step was asked, so the ask built again for the same pipeline is the same ask, digest included: a warrant is held to it.
func ApprovalAsk(p seat.Pipeline, from time.Time) (oc.Ask, error) {
sum := sha256.Sum256([]byte(p.Pipeline + "\n" + p.Commit))
act := func(do string) (string, error) {
return operatorchannelclient.ActDigest(operatorchannelclient.Act{"pipeline": p.Pipeline, "commit": p.Commit,
"fingerprint": p.Fingerprint, "do": do})
}
approve, err := act(OptionApprove)
if err != nil {
return oc.Ask{}, err
}
decline, err := act(OptionDecline)
if err != nil {
return oc.Ask{}, err
}
a := oc.Ask{ID: "approval-" + hex.EncodeToString(sum[:12]), About: About(p.Module, p.Branch), Who: operatorchannelclient.Operator,
Headline: trimRunes("Deliver "+p.Module+" from "+p.Branch+"?", operatorchannelclient.HeadlineLength),
Explanation: fmt.Sprintf("Every required step of %s's pipeline at commit %s on %s passed; its policy asks your approval before it is delivered.", p.Module, p.Commit, p.Branch),
Details: "pipeline " + p.Pipeline + ", fingerprint " + p.Fingerprint,
Expires: operatorchannelclient.FormatTime(from.Add(operatorchannelclient.ApproveLasts)),
OnExpiry: "the step ends declined and the commit is not delivered; ask-again asks once more",
Options: []oc.Option{
{ID: OptionApprove, Label: "Approve", Does: "delivers this commit of " + p.Module, Level: operatorchannelclient.Approve, Binds: approve},
{ID: OptionDecline, Label: "Decline", Does: "ends the step declined; nothing is delivered", Level: operatorchannelclient.Approve, Binds: decline},
}}
return a, operatorchannelclient.Check(a, from)
}
func trimRunes(s string, n int) string {
r := []rune(s)
if len(r) <= n {
return s
}
return string(r[:n-1]) + "…"
}
// Asker is the runner's side of the operator-channel: Ask publishes an ask under the runner's name, its duplicate id the
// ask's own, so an ask published twice is kept once.
type Asker interface {
Ask(oc.Ask) error
}
// askApproval asks p's approval step when it is due, before the pipeline is written: an ask published and then not
// written is published again, the same ask, when its step's end is heard again.
func askApproval(key string, p *seat.Pipeline, from time.Time, asker Asker) (bool, error) {
i := approvalDue(*p)
if i < 0 || asker == nil {
return false, nil
}
a, err := ApprovalAsk(*p, from)
if err != nil {
return false, fmt.Errorf("the approval of %s cannot be asked: %w", key, err)
}
if err := asker.Ask(a); err != nil {
return false, fmt.Errorf("the approval of %s is not asked: %w", key, err)
}
s := &p.Steps[i]
s.Outcome, s.Ask, s.Seat, s.Started = seat.StepResultOutcomeRunning, a.ID, seat.Seat, &from
s.Why = "waiting on the operator"
return true, nil
}
// ErrNoAsk is a warrant naming no approval step the runner keeps for its about.
var ErrNoAsk = errors.New("no approval step asked under the warrant's ask")
// decide ends an approval step on its warrant: a chosen approve, held to the ask rebuilt from the step, ends it passed; a
// chosen decline, an expiry, a cancel or a refusal ends it declined; a replacement by a newer commit's ask ends it
// superseded (design 47 G §37). It says whether the step changed.
func decide(p *seat.Pipeline, w oc.Warrant, asker string, now time.Time) (bool, error) {
for i := range p.Steps {
s := &p.Steps[i]
if s.Step != seat.StepResultStepApproval || s.Ask != w.Ask || s.Started == nil {
continue
}
if s.Outcome != seat.StepResultOutcomeRunning {
return false, nil
}
switch w.Outcome {
case oc.WarrantOutcomeChosen:
asked, err := ApprovalAsk(*p, *s.Started)
if err != nil {
return false, err
}
o, err := operatorchannelclient.For(w, asker, asked)
if err != nil {
return false, fmt.Errorf("the warrant for %s is not taken: %w", w.Ask, err)
}
s.Outcome = seat.StepResultOutcomeDeclined
if o.ID == OptionApprove {
s.Outcome = seat.StepResultOutcomePassed
}
if w.By != nil {
s.By = w.By.Identity
}
s.Why = "the operator chose " + o.ID
case oc.WarrantOutcomeReplaced:
s.Outcome, s.Why = seat.StepResultOutcomeSuperseded, "a newer commit's ask replaced it"
default:
s.Outcome, s.Why = seat.StepResultOutcomeDeclined, "the ask ended "+w.Outcome+" "+w.Words
}
s.Ended = &now
return true, nil
}
return false, ErrNoAsk
}
// ErrNoRecord is a decided event whose ask the router's record holds no warrant for yet: the event is heard again.
var ErrNoRecord = errors.New("the router's record of the ask holds no warrant yet")
// OnDecided takes the operator-channel's decided for the runner's own asks: it only wakes the runner (design 47 G A-89), which
// reads the warrant from the router's record of the ask through Where, never from the event's body (§37, §700). The approval
// step asked under that ask, among the pipelines of the module and watched branch the ask's about names, ends as decide says.
func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) {
if event.Asker != h.Name {
return seat.Pipeline{}, fmt.Errorf("%w: the decided event is %s's, not %s's", ErrRefused, event.Asker, h.Name)
}
if h.Where == nil {
return seat.Pipeline{}, fmt.Errorf("the runner reads no router's record, so the ask %s is not taken", event.Ask)
}
_, rec, err := h.Where(event.Ask)
if err != nil {
return seat.Pipeline{}, fmt.Errorf("the router's record of the ask %s cannot be read: %w", event.Ask, err)
}
if rec == nil {
return seat.Pipeline{}, fmt.Errorf("%w: %s", ErrNoRecord, event.Ask)
}
w := *rec
module, branch, ok := strings.Cut(w.About, ".")
if !ok {
return seat.Pipeline{}, fmt.Errorf("%w: the warrant for %s is about %q, not a module and a branch", ErrNoAsk, w.Ask, w.About)
}
found, err := h.Store.Keys(branch + ".*." + module)
if err != nil {
return seat.Pipeline{}, err
}
for _, key := range found {
for {
p, rev, err := h.Store.Get(key)
if err != nil {
return seat.Pipeline{}, err
}
now := h.now()
changed, err := decide(&p, w, h.Name, now)
if errors.Is(err, ErrNoAsk) {
break
}
if err != nil || !changed {
return p, err
}
ended := endPipeline(&p, now)
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return p, err
}
i := stepOfAsk(p, w.Ask)
h.logEnd(key, p.Steps[i], now)
for _, ev := range []string{seat.EventWaitEnded, seat.EventStepEnded} {
if err := h.say(ev, p, &p.Steps[i]); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", key, ev, err)
}
}
return p, h.said(p, false, ended)
}
}
return seat.Pipeline{}, fmt.Errorf("%w: %s about %s", ErrNoAsk, w.Ask, w.About)
}
@@ -0,0 +1,95 @@
package pipeline
import (
"errors"
"fmt"
"strings"
"time"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
nodebuildimage "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode"
)
// AskBound is how long an ask on a build seat's work queue may wait for its holder to take it (design 47 G §535, §647): five
// minutes, the bound of a key of the state `bounds` with its own time to live (A-77, A-104), never a timer.
const AskBound = 5 * time.Minute
// TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it.
type TakenSeat struct {
Seat string
// Subject hears every node's ask-taken of the seat.
Subject string
}
// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken,
// so no bound is armed for their asks.
var TakenSeats = []TakenSeat{
{Seat: nodebuildgo.Seat, Subject: nodebuildgo.SubjectAskTakenAll},
{Seat: nodebuildnode.Seat, Subject: nodebuildnode.SubjectAskTakenAll},
{Seat: nodebuildimage.Seat, Subject: nodebuildimage.SubjectAskTakenAll},
}
// BoundSeat says whether the holder of a seat says ask-taken, so an ask put on its work queue has a bound.
func BoundSeat(name string) bool {
for _, s := range TakenSeats {
if s.Seat == name {
return true
}
}
return false
}
// Arm puts a bound of one ask's wait for its holder: a key of the state `bounds` named for the ask, written with its own
// time to live. It is armed before the ask is put, so an ask-taken heard at once finds it.
type Arm func(ask string) error
// NotTaken ends the step asked under ask that no holder took within its bound, the bound's expiry heard (design 47 G §32,
// §535, §647, A-77): the step ends `could not run` naming the seat, so its pipeline ends failed once the others end, never
// a silent wait. A step that ended or was never asked of a seat takes nothing.
func (h *Hearer) NotTaken(ask string, bound time.Duration) error {
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return fmt.Errorf("the pipelines are not listed: %w", err)
}
for _, key := range pkeys {
p, _, err := h.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || stepOfAsk(p, ask) < 0 {
continue
}
return h.endNotTaken(key, ask, bound)
}
return nil
}
func (h *Hearer) endNotTaken(key, ask string, bound time.Duration) error {
for {
p, rev, err := h.Store.Get(key)
if err != nil {
return fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
i := stepOfAsk(p, ask)
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning {
return nil
}
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Ended = seat.StepResultOutcomeCouldNotRun, &now
s.Why = fmt.Sprintf("the seat %s has no holder that took ask %s within %s", s.Seat, ask, bound)
ended := endPipeline(&p, now)
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return fmt.Errorf("pipeline %s cannot be written: %w", key, err)
}
h.logEnd(key, *s, now)
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
}
return h.said(p, false, ended)
}
}
// IsAskKey says whether a key of the state `bounds` is an ask's bound.
func IsAskKey(key string) bool { return strings.HasPrefix(key, "step-") && !strings.Contains(key, ".") }
@@ -0,0 +1,168 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"time"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
nodebuildimage "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode"
)
// StepAsker puts one step's ask on a build seat's work queue, at the subject accept names (the seat's accepted kind `step`
// on its stream `check`, design 47 G §22, §24). An ask asked twice under one id is kept once.
type StepAsker interface {
AskStep(accept, id string, body any) error
}
// acceptStep is where each build seat is sent its step, from its contract.
var acceptStep = map[string]string{
builds.Seat: builds.AcceptSubjectStep,
wants.Seat: wants.AcceptSubjectStep,
nodebuildgo.Seat: nodebuildgo.AcceptSubjectCheckStep,
nodebuildnode.Seat: nodebuildnode.AcceptSubjectCheckStep,
nodebuildimage.Seat: nodebuildimage.AcceptSubjectCheckStep,
}
// buildSteps are the steps an artifact's build seat runs, in the order they are asked (design 47 G §22).
var buildSteps = []string{seat.StepResultStepLint, seat.StepResultStepUnit, seat.StepResultStepIntegration}
// moduleSteps are the steps of the whole module: composition (the assigner's), review (the runner's own verb) and the
// approval, asked last (§22, §26).
var moduleSteps = []string{seat.StepResultStepComposition, seat.StepResultStepReview, seat.StepResultStepApproval}
// kindImage is the form of an acting rule's `on` naming an artifact kind (§324).
const kindImage = "kind: image"
// AskID is a step's ask id, the same for the same pipeline, artifact and step, so a start asked again asks nothing new.
func AskID(pipeline, artifact, step string) string {
sum := sha256.Sum256([]byte(pipeline + "\n" + artifact + "\n" + step))
return "step-" + hex.EncodeToString(sum[:12])
}
// Place places a pipeline's steps (design 47 G §21, §23, §323; A-91): only the steps the module's registration policy
// names, each artifact's lint, unit and integration steps beside the module's composition, review and approval. A step the
// policy does not name is no step. A `bypass` step is placed bypassed and asked of no one; a build step is placed running,
// asked of the build seat the artifact's language names in the catalog half (§323), or, where no seat names it, ended
// could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats'
// work queues, one per running step.
func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) {
entry := entries[p.Module]
arts := append([]catalog.Artifact(nil), entry.Artifacts...)
sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref })
var steps []seat.StepResult
var asks []Asked
// The build step is every pipeline's first, required whatever the policy names, asked of the seat mesh-builds for the
// module's own entry only (A-99, A-97): a group's other members are built by their own pipelines.
build := seat.StepResult{Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Fingerprint: p.Fingerprint,
Seat: builds.Seat, Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: AskID(p.Pipeline, "", seat.StepResultStepBuild)}
steps = append(steps, build)
asks = append(asks, Asked{Seat: builds.Seat, ID: build.Ask, Ask: builds.StepRun{Pipeline: p.Pipeline, Step: builds.StepRunStepBuild, Branch: p.Branch,
Revision: p.Revision, Source: p.Source, Entries: []builds.BuildEntry{{Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint}}}})
for _, a := range arts {
name := a.Ref
if _, after, ok := strings.Cut(a.Ref, ":"); ok {
name = after
}
for _, step := range buildSteps {
status := Status(reg, name, step)
if status == "" {
continue
}
s := seat.StepResult{Step: step, Artifact: a.Ref, Status: status, Fingerprint: p.Fingerprint}
if status == seat.StepResultStatusBypass {
s.Outcome, s.Why = seat.StepResultOutcomeBypassed, "bypassed by the module's policy"
steps = append(steps, s)
continue
}
on := a.Language
if on == "" && a.Kind == "image" {
on = kindImage
}
bs, err := BuildSeat(entries, on)
if err != nil {
s.Outcome, s.Started, s.Ended, s.Why = seat.StepResultOutcomeCouldNotRun, &now, &now, err.Error()
steps = append(steps, s)
continue
}
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
s.Ask = AskID(p.Pipeline, a.Ref, step)
steps = append(steps, s)
ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)}
if a.Package != nil {
ask.Package = a.Package.Name
}
asks = append(asks, ask)
}
}
for _, step := range moduleSteps {
status := Status(reg, "", step)
if status == "" {
continue
}
s := seat.StepResult{Step: step, Status: status, Outcome: seat.StepResultOutcomeWaiting}
if status == seat.StepResultStatusBypass {
s.Outcome, s.Why = seat.StepResultOutcomeBypassed, "bypassed by the module's policy"
}
// Composition is the assigner's, an accepted kind step of the seat mesh-wants (§22): it is asked with the others,
// running at once, and its end is heard from that seat as the build seats' are. Review and approval wait.
if step == seat.StepResultStepComposition && status != seat.StepResultStatusBypass {
s.Seat, s.Outcome, s.Started, s.Fingerprint = wants.Seat, seat.StepResultOutcomeRunning, &now, p.Fingerprint
s.Ask = AskID(p.Pipeline, "", step)
asks = append(asks, Asked{Seat: wants.Seat, ID: s.Ask, Ask: wants.StepRun{Branch: p.Branch, Commit: p.Commit, Fingerprint: p.Fingerprint,
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: wants.StepRunStepComposition}})
}
steps = append(steps, s)
}
return steps, asks
}
// Asked is one ask to put on a build seat's work queue.
type Asked struct {
Seat string
ID string
Ask any
// Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results
// are the ask's (§25, §416).
Package string
}
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
fetch := nodebuildgo.Fetch{Repository: p.Repository, Commit: p.Commit, Dir: e.Path}
return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: a.Language, At: &now,
Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: p.Commit, Fetch: fetch, Fingerprint: p.Fingerprint,
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
}
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put.
func askAll(asker StepAsker, arm Arm, key string, asks []Asked) error {
var first error
for _, a := range asks {
accept, ok := acceptStep[a.Seat]
err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
if ok {
// A build seat's holder says ask-taken: the ask's bound is armed before it is put (§535, §647).
if arm != nil && BoundSeat(a.Seat) {
if aerr := arm(a.ID); aerr != nil {
err = fmt.Errorf("its bound is not armed: %w", aerr)
ok = false
}
}
}
if ok {
err = asker.AskStep(accept, a.ID, a.Ask)
}
if err != nil && first == nil {
first = fmt.Errorf("the step %s of %s is not asked of %s: %w", a.ID, key, a.Seat, err)
}
}
return first
}
@@ -0,0 +1,268 @@
package pipeline
import (
"encoding/json"
"strings"
"testing"
"time"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
)
func stepSeatOf(t *testing.T, name string) StepSeat {
t.Helper()
for _, bs := range StepSeats {
if bs.Seat == name {
return bs
}
}
t.Fatalf("no step seat %s", name)
return StepSeat{}
}
type stepAsks struct {
at []string
ids []string
asks []any
}
func (s *stepAsks) AskStep(accept, id string, body any) error {
s.at, s.ids, s.asks = append(s.at, accept), append(s.ids, id), append(s.asks, body)
return nil
}
func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) {
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{
Steps: catalog.StepStatuses{Lint: "required", Unit: "optional", Approval: "required"},
Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}},
}}
return reg, map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"},
{Ref: "app:web", Language: "rust"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
}
}
// The runner's asking half (design 47 G §22, §24, §323; A-91): a pipeline's steps are the ones the policy names, a bypassed
// step asked of no one, an artifact no build seat names ended could not run naming it, and each running step asked of its
// build seat's accepted kind step with the pipeline's commit and fingerprint; a step the policy does not name is no step.
func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) {
reg, entries := placeFixture()
store := created{}
asks := &stepAsks{}
s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
var p seat.Pipeline
if err := json.Unmarshal(store["main.4.app"], &p); err != nil {
t.Fatal(err)
}
var got []string
for _, st := range p.Steps {
got = append(got, st.Artifact+" "+st.Step+" "+st.Outcome)
}
want := []string{" build running", "app:program lint running", "app:program unit running", "app:tools lint bypassed", "app:tools unit running",
"app:web lint could not run", "app:web unit could not run", " approval waiting"}
if strings.Join(got, "|") != strings.Join(want, "|") {
t.Errorf("steps = %q, want %q (integration and composition are named by no policy: no step)", got, want)
}
if !strings.Contains(p.Steps[5].Why, `"rust"`) {
t.Errorf("the step of an artifact no seat builds says %q, want it to name the language", p.Steps[5].Why)
}
if len(asks.asks) != 4 {
t.Fatalf("asked %d steps, want the build and the three running ones", len(asks.asks))
}
// A-99: the build is every pipeline's first step, required whatever the policy names, asked of mesh-builds for the
// module's own entry only.
b, ok := asks.asks[0].(builds.StepRun)
if !ok || asks.at[0] != builds.AcceptSubjectStep || p.Steps[0].Status != seat.StepResultStatusRequired || asks.ids[0] != p.Steps[0].Ask ||
len(b.Entries) != 1 || b.Entries[0] != (builds.BuildEntry{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}) || b.Pipeline != "main.4.app" {
t.Errorf("the build ask = %+v at %s", asks.asks[0], asks.at[0])
}
a := asks.asks[1].(nodebuildgo.StepAsk)
if asks.at[1] != nodebuildgo.AcceptSubjectCheckStep || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID ||
a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4.app" || a.Step.Artifact != "app:program" ||
a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 {
t.Errorf("the ask = %+v at %s", a, asks.at[1])
}
// A second start of the revision asks nothing.
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Entries: entries}); ok || err != nil || len(asks.asks) != 4 {
t.Errorf("second start = %v, %v, %d asks", ok, err, len(asks.asks))
}
}
// A pipeline whose other steps are all bypassed still has its build, required whatever the policy names, so its approval is
// not due at its start (A-99, §26).
func TestTheBuildHoldsTheApprovalWhateverThePolicyNames(t *testing.T) {
reg, entries := placeFixture()
reg.Policy = catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "bypass", Approval: "required"}}
store := created{}
s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: &stepAsks{},
Registration: func(string) (catalog.Registration, bool) { return reg, true }}
if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); err != nil {
t.Fatal(err)
}
var p seat.Pipeline
_ = json.Unmarshal(store["main.4.app"], &p)
if p.Steps[0].Step != seat.StepResultStepBuild || p.Steps[0].Status != seat.StepResultStatusRequired || approvalDue(p) != -1 {
t.Errorf("steps = %+v; want the build first, required, and no approval due", p.Steps)
}
}
// The seat mesh-builds' end of the build step is taken as the others are, held to its one entry's module, commit and
// fingerprint; an end naming two entries is refused (A-97, A-99).
func TestTheBuildStepsEndIsTakenOnItsOneEntry(t *testing.T) {
bs := stepSeatOf(t, builds.Seat)
h, store, _, _, _, _ := fixture()
e := store[pkey]
e.p.Steps = append([]seat.StepResult{{Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning,
Ask: "b1", Seat: builds.Seat}}, e.p.Steps...)
store[pkey] = e
raw := `{"id":"b1","pipeline":"main.5.app","step":"build","node":"n1","outcome":"passed","entries":[{"module":"app","commit":"c1","fingerprint":"src2:aa"}]}`
ev, err := bs.Decode([]byte(raw))
if err != nil {
t.Fatal(err)
}
p, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev)
if err != nil || p.Steps[0].Outcome != seat.StepResultOutcomePassed {
t.Fatalf("the build end: %v, step %+v", err, p.Steps[0])
}
if _, err := bs.Decode([]byte(`{"id":"b1","entries":[{"module":"a","fingerprint":"x"},{"module":"b","fingerprint":"y"}]}`)); err == nil {
t.Error("an end of two entries is refused")
}
}
// Composition is asked of the seat mesh-wants with the other steps, running at once, and its end is heard from that seat
// (design 47 G §22, §24): a pipeline whose policy names it places it running with its own ask, a bypassed one asks no one.
func TestCompositionIsAskedOfTheAssignerAndItsEndIsTaken(t *testing.T) {
reg, entries := placeFixture()
reg.Policy = catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "bypass", Composition: "required", Approval: "required"}}
p := seat.Pipeline{Pipeline: "main.4.app", Branch: "main", Module: "app", Commit: "c1", Fingerprint: "src2:aa", Revision: 7, Source: 4}
steps, asked := Place(p, reg, entries, time.Unix(1, 0), "the-runner")
var comp seat.StepResult
for _, s := range steps {
if s.Step == seat.StepResultStepComposition {
comp = s
}
}
if comp.Outcome != seat.StepResultOutcomeRunning || comp.Seat != wants.Seat || comp.Ask == "" {
t.Fatalf("composition = %+v, want running, asked of mesh-wants", comp)
}
var got *Asked
for i := range asked {
if asked[i].Seat == wants.Seat {
got = &asked[i]
}
}
r, ok := got.Ask.(wants.StepRun)
if got == nil || !ok || got.ID != comp.Ask || acceptStep[wants.Seat] != wants.AcceptSubjectStep || r.Step != "composition" ||
r.Commit != "c1" || r.Fingerprint != "src2:aa" || r.Pipeline != "main.4.app" || r.Revision != 7 || r.Source != 4 || r.Module != "app" {
t.Fatalf("the composition ask = %+v", got)
}
// Its end, from the seat's own subject, is taken like the others, a node's warning becoming the step's why.
bs := stepSeatOf(t, wants.Seat)
ev, err := bs.Decode([]byte(`{"id":"` + comp.Ask + `","pipeline":"main.5.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed","warnings":["n2 asleep"]}`))
if err != nil || ev.Said != "n2 asleep" || ev.Outcome != "passed" || ev.Module != "app" {
t.Fatalf("decoded end = %+v, %v", ev, err)
}
}
// A runner restarted after it wrote a pipeline and before its asks reached the queues asks them again when the branch's
// view holds its half: each step placed and still running, of a running pipeline of that branch, under the id it was
// placed under; an ended step, an ended pipeline and another branch's pipeline are asked of no one (design 47 G §88).
func TestResumeAsksTheStepsPlacedAndNotEnded(t *testing.T) {
reg, entries := placeFixture()
placed := func(branch string, source int64) seat.Pipeline {
p := seat.Pipeline{Pipeline: Key(branch, source, "app"), Branch: branch, Source: source, Module: "app", Commit: "c1", Fingerprint: "src2:aa",
Repository: "o/app", Outcome: seat.PipelineOutcomeRunning}
p.Steps, _ = Place(p, reg, entries, time.Unix(1, 0), "the-runner")
return p
}
main4, main5, beta6 := placed("main", 4), placed("main", 5), placed("beta", 6)
main5.Steps[1].Outcome = seat.StepResultOutcomePassed // the build of main.5 ended
ended := placed("main", 3)
ended.Outcome = seat.PipelineOutcomePassed
store := memStore{}
for _, p := range []seat.Pipeline{main4, main5, beta6, ended} {
store[p.Pipeline] = struct {
p seat.Pipeline
rev uint64
}{p, 1}
}
asks := &stepAsks{}
s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
n, err := s.Resume(store, "main", Half{Entries: entries})
// main.4 has the build and three running steps of the fixture; main.5 one fewer, its build ended.
if err != nil || n != 7 || len(asks.asks) != 7 {
t.Fatalf("resume asked %d (%d put), %v; want 7", n, len(asks.asks), err)
}
for _, id := range asks.ids {
found := false
for _, p := range []seat.Pipeline{main4, main5} {
for _, st := range p.Steps {
found = found || (st.Ask == id && st.Outcome == seat.StepResultOutcomeRunning)
}
}
if !found {
t.Errorf("ask %s is of no running step of the branch's running pipelines", id)
}
}
}
// A step ask carries the kept results of its artifact's package and the toolchain of its language (design 47 G §25, §416,
// A-104): the results whose package is the artifact's package or lies beneath it, read once for all the asks; a failed
// read asks the steps with none.
func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) {
reg, entries := placeFixture()
e := entries["app"]
e.Artifacts[0].Package = &catalog.ArtifactPackage{Ecosystem: "go", Name: "git.example/o/app"}
entries["app"] = e
store, asks := created{}, &stepAsks{}
reads := 0
s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
Registration: func(string) (catalog.Registration, bool) { return reg, true },
Kept: func(pkgs []string) (map[string][]nodebuildgo.KeptResult, error) {
reads++
if len(pkgs) != 1 || pkgs[0] != "git.example/o/app" {
t.Errorf("kept read for %v, want the one package of the entry", pkgs)
}
return map[string][]nodebuildgo.KeptResult{pkgs[0]: {{Digest: "sha256:aa", Level: "unit", Package: "git.example/o/app/sub"}}}, nil
},
Toolchain: func(l string) (*nodebuildgo.ToolchainUsed, error) {
return &nodebuildgo.ToolchainUsed{Language: l, Version: "1.26"}, nil
}}
if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); err != nil {
t.Fatal(err)
}
var program, tools *nodebuildgo.StepRun
for _, a := range asks.asks {
if ask, ok := a.(nodebuildgo.StepAsk); ok {
switch {
case ask.Step.Artifact == "app:program" && ask.Step.Step == "lint":
program = ask.Step
case ask.Step.Artifact == "app:tools" && ask.Step.Step == "unit":
tools = ask.Step
}
}
}
if reads != 1 || program == nil || len(program.Kept) != 1 || program.Kept[0].Digest != "sha256:aa" || program.Toolchain == nil || program.Toolchain.Version != "1.26" {
t.Errorf("the program's ask = %+v after %d reads", program, reads)
}
if tools == nil || len(tools.Kept) != 0 || tools.Toolchain == nil {
t.Errorf("an artifact with no package carries no kept result and its language's toolchain: %+v", tools)
}
for pkg, want := range map[string]bool{"a/b": true, "a/b/c": true, "a/bc": false, "a": false} {
if InPackage(pkg, "a/b") != want {
t.Errorf("InPackage(%s, a/b) = %v, want %v", pkg, !want, want)
}
}
if tc, err := ToolchainOf([]byte(`{"language":"go","version":"1.26.1","digests":[]}`)); err != nil || tc.Language != "go" || tc.Version != "1.26.1" {
t.Errorf("ToolchainOf = %+v, %v", tc, err)
}
}
@@ -12,7 +12,7 @@ import (
// Status is a step's status for one artifact of a module, read from the module's registration and never from a
// manifest (design 47 G §21, §23): the artifact's own status where its policy names one, else the module's. A step
// neither names is "" (design question Q-RN8, open).
// neither names is "" and is no step of the pipeline (A-91): Place places none for it.
func Status(reg catalog.Registration, artifact, step string) string {
if a, ok := reg.Policy.Artifacts[artifact]; ok {
if s := statusOf(a, step); s != "" {
@@ -81,6 +81,9 @@ type BranchView interface {
View() (Half, bool)
}
// readyView is a BranchView that says when it holds its first half, as meshcatalogclient.Catalog does.
type readyView interface{ Ready() <-chan struct{} }
// Half is the catalog half a branch view holds, as the runner reads it: its revision and its entries by module.
type Half struct {
Revision int64
@@ -95,7 +98,10 @@ type Views struct {
// Open makes and starts the view of a branch; its error is said in the journal and the branch asked again at the
// next registration heard.
Open func(branch string) (BranchView, error)
Logf func(string, ...any)
// OnReady is told once for a branch, when its view holds a first half: the steps placed in the pipelines of that branch
// and not yet asked are asked then (§88's restart pickup).
OnReady func(branch string)
Logf func(string, ...any)
mu sync.Mutex
branches map[string]BranchView
@@ -126,6 +132,13 @@ func (v *Views) ensure(branch string) {
return
}
v.branches[branch] = bv
if r, ok := bv.(readyView); ok && v.OnReady != nil {
on := v.OnReady
go func() {
<-r.Ready()
on(branch)
}()
}
}
// Branches are the watched branches the views hold, sorted.
@@ -83,3 +83,33 @@ type fakeView struct {
func (f fakeView) Hear(context.Context, []byte) error { f.heard[f.branch]++; return nil }
func (f fakeView) View() (Half, bool) { return Half{Revision: 7}, true }
type readyFake struct {
fakeView
ch chan struct{}
}
func (r readyFake) Ready() <-chan struct{} { return r.ch }
// A branch's view that holds its first half tells Views once, so the runner asks the steps placed on that branch (§88).
func TestViewsTellOnceWhenABranchHoldsItsFirstHalf(t *testing.T) {
ch := make(chan struct{})
told := make(chan string, 4)
v := &Views{OnReady: func(b string) { told <- b }, Open: func(b string) (BranchView, error) {
return readyFake{fakeView{branch: b, heard: map[string]int{}}, ch}, nil
}}
v.Registered(catalog.Registration{Module: "n", Branches: []string{"main"}})
v.Registered(catalog.Registration{Module: "m", Branches: []string{"main"}})
select {
case b := <-told:
t.Fatalf("told %s before the view held a half", b)
default:
}
close(ch)
if b := <-told; b != "main" {
t.Errorf("told %q, want main", b)
}
if len(told) != 0 {
t.Error("a branch's readiness is told once")
}
}
@@ -11,6 +11,7 @@ import (
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/keys"
)
@@ -34,6 +35,8 @@ type Begin struct {
Group string
Repository string
Revision int64
// Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them.
Entries map[string]catalog.CatalogEntry
}
// Key is a pipeline's key in the state pipelines, `<branch>.<source revision>.<module>`, the branch escaped as a key's
@@ -51,6 +54,22 @@ type Starter struct {
Say func(subject string, body any) error
Now func() time.Time
Logf func(string, ...any)
// Registration is a module's registration, whose policy names the steps of its pipeline (§21, §23); with none, a
// pipeline starts with no step and says so.
Registration func(module string) (catalog.Registration, bool)
// Steps asks each build seat for its step, as the pipeline's asking half (§22, §24); Name is the runner's own name as an
// asker.
Steps StepAsker
Name string
// Kept reads the kept results of each named package, by the package name an artifact's catalog entry carries (§25, §416): the
// results whose package is that name or one beneath it. A step ask carries them, so a build seat runs only the packages
// with no result for their digest.
Kept func(packages []string) (map[string][]nodebuildgo.KeptResult, error)
// Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when
// none was heard, which asks the step with none.
Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error)
// Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none.
Arm Arm
}
// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second
@@ -67,6 +86,14 @@ func (s *Starter) Start(b Begin) (bool, error) {
p := seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fingerprint: b.Fp,
Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now,
Steps: []seat.StepResult{}}
var asks []Asked
if s.Registration != nil {
if reg, ok := s.Registration(b.Module); ok {
p.Steps, asks = Place(p, reg, b.Entries, now, s.Name)
} else {
s.logf("%s has no registration yet: its pipeline starts with no step", key)
}
}
raw, err := json.Marshal(p)
if err != nil {
return false, err
@@ -86,9 +113,116 @@ func (s *Starter) Start(b Begin) (bool, error) {
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks); err != nil {
return true, err
}
}
return true, nil
}
// Resume asks again, of the build seats and the assigner, each step of branch's running pipelines that is placed and still
// running (design 47 G §88's restart pickup): a pipeline written whose asks never reached their queues (the runner stopped
// between its write and its asks, or a publish failed) is asked at the next start. Each ask carries the id its step was
// placed under, so one the seat holds already is kept once. half is the branch's catalog half, read for the steps' artifacts
// and seats as Place reads it. It says how many steps it asked.
func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
if s.Steps == nil || s.Registration == nil {
return 0, nil
}
pkeys, err := store.Keys("*.*.*")
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
}
asked := 0
var first error
for _, key := range pkeys {
p, _, err := store.Get(key)
if err != nil {
if first == nil {
first = fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
continue
}
if p.Branch != branch || p.Outcome != seat.PipelineOutcomeRunning {
continue
}
reg, ok := s.Registration(p.Module)
if !ok {
continue
}
running := map[string]bool{}
for _, st := range p.Steps {
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
running[st.Ask] = true
}
}
_, placed := Place(p, reg, half.Entries, now, s.Name)
var due []Asked
for _, a := range placed {
if running[a.ID] {
due = append(due, a)
}
}
if len(due) == 0 {
continue
}
s.carry(key, due)
if err := askAll(s.Steps, s.Arm, key, due); err != nil && first == nil {
first = err
}
asked += len(due)
}
return asked, first
}
// carry puts the kept results of each ask's package and the toolchain of its language on its step ask (§25, §416, A-104). A
// read that fails asks the steps with none, so every package runs: slower, never wrong; it is said.
func (s *Starter) carry(key string, asks []Asked) {
var pkgs []string
seen := map[string]bool{}
for _, a := range asks {
if a.Package != "" && !seen[a.Package] {
seen[a.Package] = true
pkgs = append(pkgs, a.Package)
}
}
var kept map[string][]nodebuildgo.KeptResult
if s.Kept != nil && len(pkgs) > 0 {
var err error
if kept, err = s.Kept(pkgs); err != nil {
s.logf("the kept results for the steps of %s are not read, so each package runs: %v", key, err)
}
}
chains := map[string]*nodebuildgo.ToolchainUsed{}
for _, a := range asks {
ask, ok := a.Ask.(nodebuildgo.StepAsk)
if !ok || ask.Step == nil {
continue
}
if a.Package != "" {
ask.Step.Kept = kept[a.Package]
}
if s.Toolchain == nil || ask.Language == "" {
continue
}
t, done := chains[ask.Language]
if !done {
var err error
if t, err = s.Toolchain(ask.Language); err != nil {
s.logf("the toolchain of %s for the steps of %s is not read: %v", ask.Language, key, err)
}
chains[ask.Language] = t
}
ask.Step.Toolchain = t
}
}
func (s *Starter) logf(format string, args ...any) {
if s.Logf != nil {
s.Logf(format, args...)
@@ -118,7 +252,7 @@ func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.Ou
if o.Kind != outdatedKind || o.Module == "" {
continue
}
b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision}
b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision, Entries: half.Entries}
if e, ok := half.Entries[o.Module]; ok {
b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository
} else {
@@ -134,3 +268,35 @@ func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.Ou
}
return started, first
}
// TakenInSubject is where any module's take-in.taken is said, `mesh.mod.<module>.event.take-in.taken`: the runner names no
// module (§7), so the bus's grant from the manifest's `consumes` says which one it hears (§54).
const TakenInSubject = "mesh.mod.*.event.take-in.taken"
// TakenIn starts, for a take-in.taken, one pipeline per entry of the source revision, keyed by its watched branch, that
// source revision and the module (design 47 G §44, §54, §476; ADR 0666 §13). A hand drop takes nothing in and starts
// nothing; a second start of an entry, a package-published having started it first, writes nothing (§54). view is the
// catalog half of the event's branch, read only for the revision its pipelines are made from.
func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half, bool)) (int, error) {
if ev.Dropped {
s.logf("the take-in %s was dropped by %s: it starts no pipeline", ev.Key, ev.By)
return 0, nil
}
if ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a take-in.taken names its watched branch and source revision: key %q branch %q source %d", ev.Key, ev.Branch, ev.Source)
}
half, _ := view(ev.Branch)
started := 0
var first error
for _, e := range ev.Entries {
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint,
Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries})
if err != nil && first == nil {
first = err
}
if ok {
started++
}
}
return started, first
}
@@ -85,3 +85,31 @@ func TestStarts(t *testing.T) {
type failing struct{}
func (failing) Create(string, []byte) error { return errors.New("bus away") }
// A take-in.taken starts one pipeline per entry of its source revision (design 47 G §44, §54, §476); a hand drop starts
// none, and a publish that started an entry first leaves the take-in's start of it writing nothing.
func TestTakenInStarts(t *testing.T) {
store := created{}
var said []string
s := &Starter{Pipelines: store, Say: func(subject string, body any) error { said = append(said, subject); return nil }}
view := func(string) (Half, bool) { return Half{Revision: 12}, true }
ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 7, Group: "g1", Entries: []catalog.TakeInEntry{
{Module: "a", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/a"}, {Module: "b", Commit: "c2", Fingerprint: "src2:bb"}}}
if _, err := s.Start(Begin{Branch: "main", Source: 7, Module: "b", Commit: "c2"}); err != nil {
t.Fatal(err)
}
n, err := s.TakenIn(ev, view)
if err != nil || n != 1 || len(said) != 2 {
t.Fatalf("a take-in started %d (%v), said %v: want a only, b being started already", n, err, said)
}
var p seat.Pipeline
if err := json.Unmarshal(store["main.7.a"], &p); err != nil || p.Commit != "c1" || p.Group != "g1" || p.Revision != 12 || p.Fingerprint != "src2:aa" {
t.Errorf("the pipeline main.7.a = %+v (%v)", p, err)
}
if n, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Dropped: true, By: "op", Branch: "main", Source: 8, Entries: ev.Entries}, view); n != 0 || err != nil {
t.Errorf("a hand drop started %d (%v)", n, err)
}
if _, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Branch: "main"}, view); err == nil {
t.Error("a take-in naming no source revision is refused")
}
}
@@ -0,0 +1,313 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
nodebuildimage "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// StepSeat is a seat whose holder runs the lint and test steps (design 47 G §22, §24): where the runner hears its
// step-ended, on the subject of the node that ran it.
type StepSeat struct {
Seat string
// Prefix is the subject before <node>.
Prefix string
// Decode reads the seat's step-ended body; nil is the build seats' one form.
Decode func([]byte) (StepEnded, error)
}
// StepSeats are the three build seats, from their contracts.
var StepSeats = []StepSeat{
{Seat: nodebuildgo.Seat, Prefix: strings.TrimSuffix(nodebuildgo.SubjectStepEndedAll, "*")},
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
// The seat mesh-wants answers the composition step (§22, §24).
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
}
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
// warning of a node asleep (a pass with a warning, ADR 0593 §9) is the step's `why` when the end says nothing else.
func decodeWantsEnded(raw []byte) (StepEnded, error) {
var w wants.StepEnded
if err := json.Unmarshal(raw, &w); err != nil {
return StepEnded{}, err
}
said := w.Said
if said == "" && len(w.Warnings) > 0 {
said = strings.Join(w.Warnings, "; ")
}
return StepEnded{V: w.V, ID: w.ID, Pipeline: w.Pipeline, Step: w.Step, Module: w.Module, Commit: w.Commit, Fingerprint: w.Fingerprint,
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
}
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
// refused (A-99).
func decodeBuildEnded(raw []byte) (StepEnded, error) {
var b builds.StepEnded
if err := json.Unmarshal(raw, &b); err != nil {
return StepEnded{}, err
}
if len(b.Entries) != 1 {
return StepEnded{}, fmt.Errorf("%w: ask %s ended a build of %d entries, asked of one", ErrRefused, b.ID, len(b.Entries))
}
e := b.Entries[0]
return StepEnded{V: b.V, ID: b.ID, Pipeline: b.Pipeline, Step: b.Step, Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint,
Node: b.Node, Outcome: b.Outcome, Said: b.Said, Rerun: b.Rerun, By: b.By, Started: b.Started, Ended: b.Ended}, nil
}
// StepEnded is a build seat's step-ended. The three seats' bodies are one form, the part build-step's, held equal by
// their definitions, so the runner reads each in this one type.
type StepEnded = nodebuildgo.StepEnded
// ErrRefused is a step-ended the runner does not take: not the node's holder's subject, or naming an ask, a commit or a
// fingerprint the runner did not ask (ADR 0666 §21; design 47 G §24). It is said, never taken.
var ErrRefused = errors.New("a step's end the runner does not take")
// ErrChanged is an Updater's answer to an Update whose revision is no longer the key's: another write came between.
var ErrChanged = errors.New("the pipeline was written since it was read")
// Updater is the state pipelines read and written by compare-and-set on the revision read (§27).
type Updater interface {
Get(key string) (seat.Pipeline, uint64, error)
Update(key string, p seat.Pipeline, revision uint64) (uint64, error)
// Keys are the keys matching a filter, `*` one key part.
Keys(filter string) ([]string, error)
}
// Hearer takes the steps' ends into their pipelines and ends a pipeline whose every step ended (design 47 G §24, §27).
type Hearer struct {
Store Updater
// Say says one of the runner's events of a pipeline, on its subject; Log appends to the step log.
Say func(subject string, body any) error
Log func(seat.LogEntry) error
// Keep writes one package's passing result into the state kept, under its key (§27): only from a step's end the
// runner took, so a kept result is its seat holder's alone.
Keep func(key string, k seat.KeptResult) error
// Output keeps a step's output under key in the objects step-outputs.
Output func(key string, data []byte) error
// Asker publishes an approval step's ask (§37); Name is the runner's own name, the asker a warrant names.
Asker Asker
Name string
// Where reads the router's record of one of the runner's asks, as operatorchannelclient.Client.Where does: its state and
// its warrant (design 47 G A-89, §700).
Where func(ask string) (string, *oc.Warrant, error)
Logf func(string, ...any)
Now func() time.Time
}
func (h *Hearer) now() time.Time {
if h.Now != nil {
return h.Now().UTC()
}
return time.Now().UTC()
}
// OnStepEnded takes the step-ended heard on subject, from seat's holders. It is refused, and said, when the subject's
// node is not the one the body names, when no pipeline the runner keeps names the ask, or when the ask's step names
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
node := strings.TrimPrefix(subject, bs.Prefix)
if node == subject || node == "" || strings.Contains(node, ".") {
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
}
if ev.Node != node {
return seat.Pipeline{}, fmt.Errorf("%w: ask %s ended on %s's subject names node %q", ErrRefused, ev.ID, node, ev.Node)
}
for {
p, rev, err := h.Store.Get(ev.Pipeline)
if err != nil {
return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err)
}
i := stepOfAsk(p, ev.ID)
switch {
case i < 0:
return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID)
case p.Steps[i].Seat != bs.Seat:
return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat)
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
return p, nil
case !endOutcome(ev.Outcome):
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
}
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
if ev.Tests != nil {
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
if ev.Output != "" && h.Output != nil {
name := seat.StepOutputsByPipelineStepArtifactKey(keys.EscapeName(ev.Pipeline), s.Step, keys.EscapeName(s.Artifact))
if err := h.Output(name, []byte(ev.Output)); err != nil {
return p, fmt.Errorf("the output of ask %s cannot be kept: %w", ev.ID, err)
}
s.Output = name
}
from := now
if ev.Ended != nil {
from = ev.Ended.UTC()
}
asked, err := askApproval(ev.Pipeline, &p, from, h.Asker)
if err != nil {
return p, err
}
ended := endPipeline(&p, now)
if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
}
if h.Keep != nil {
for _, k := range ev.Kept {
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline}
if k.Toolchain != nil {
kept.Toolchain = &seat.ToolchainUsed{Language: k.Toolchain.Language, Recipe: k.Toolchain.Recipe, Version: k.Toolchain.Version}
}
if kept.Pipeline == "" {
kept.Pipeline = ev.Pipeline
}
if err := h.Keep(seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)), kept); err != nil {
h.logf("the passing result of %s (%s) at %s is not kept, so it runs again next time: %v", k.Package, k.Level, k.Digest, err)
}
}
}
h.logEnd(ev.Pipeline, *s, now)
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
}
return p, h.said(p, asked, ended)
}
}
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
if asked {
if err := h.say(seat.EventWaitHeld, p, nil); err != nil {
return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err)
}
}
if ended {
if err := h.say(seat.EventPipelineEnded, p, nil); err != nil {
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
}
}
return nil
}
func (h *Hearer) logEnd(key string, s seat.StepResult, at time.Time) {
if h.Log == nil {
return
}
if err := h.Log(seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
Output: s.Output, At: at}); err != nil {
h.logf("the step log misses the end of %s of %s: %v", s.Step, key, err)
}
}
func (h *Hearer) logf(format string, args ...any) {
if h.Logf != nil {
h.Logf(format, args...)
}
}
// say says one event of p on the runner's subject for it; step, when not nil, is the result a step-ended carries.
func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error {
last := p.Commit
if p.Group != "" {
last = p.Group
}
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
at := h.now()
switch event {
case seat.EventStepEnded, seat.EventStepStarted:
return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step})
case seat.EventPipelineEnded:
return h.Say(subject, pipelineEnded(p))
default:
return h.Say(subject, seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
Group: p.Group, Revision: p.Revision, Source: p.Source, Step: seat.StepResultStepApproval, On: "operator", Since: at})
}
}
func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
return seat.PipelineEnded{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
Group: p.Group, Revision: p.Revision, Source: p.Source, Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started,
Ended: p.Ended, EndCount: p.EndCount, Steps: p.Steps}
}
func endOutcome(o string) bool {
switch o {
case nodebuildgo.StepEndedOutcomePassed, nodebuildgo.StepEndedOutcomeFailed, nodebuildgo.StepEndedOutcomeNothingToRun,
nodebuildgo.StepEndedOutcomeCouldNotRun:
return true
}
return false
}
func stepOfAsk(p seat.Pipeline, id string) int {
for i, s := range p.Steps {
if id != "" && s.Ask == id {
return i
}
}
return -1
}
// endPipeline ends a pipeline whose every step ended: passed when every required step passed or was carried, else
// failed, naming each required step that held it (ADR 0666 §7, §9; design 47 G §24, §26). A failed optional step, and an optional step
// still waiting, hold nothing. An approval step still waiting when a required step did not pass is `not asked` (§26). It says whether the
// pipeline ended now, its end count one higher.
func endPipeline(p *seat.Pipeline, now time.Time) bool {
if p.Outcome != seat.PipelineOutcomeRunning {
return false
}
var held []string
notAsked := -1
for i, s := range p.Steps {
switch {
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
notAsked = i
continue
case s.Outcome == seat.StepResultOutcomeWaiting && s.Status != seat.StepResultStatusRequired:
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
// pipeline's end (A-104).
continue
case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning:
return false
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
held = append(held, s.Artifact+" "+s.Step+" "+s.Outcome)
}
}
if notAsked >= 0 {
if len(held) == 0 {
return false
}
p.Steps[notAsked].Outcome = seat.StepResultOutcomeNotAsked
}
p.Ended = &now
p.EndCount++
p.Outcome, p.Why = seat.PipelineOutcomePassed, ""
if len(held) > 0 {
p.Outcome, p.Why = seat.PipelineOutcomeFailed, "required steps not passed: "+strings.Join(held, ", ")
}
return true
}
@@ -0,0 +1,350 @@
package pipeline
import (
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/keys"
)
type memStore map[string]struct {
p seat.Pipeline
rev uint64
}
func (m memStore) Get(key string) (seat.Pipeline, uint64, error) {
e, ok := m[key]
if !ok {
return seat.Pipeline{}, 0, errors.New("no such key")
}
// A read is a copy, as one from the bus is: a change not written is not seen by the next read.
e.p.Steps = append([]seat.StepResult(nil), e.p.Steps...)
return e.p, e.rev, nil
}
func (m memStore) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) {
if m[key].rev != rev {
return 0, ErrChanged
}
m[key] = struct {
p seat.Pipeline
rev uint64
}{p, rev + 1}
return rev + 1, nil
}
func (m memStore) Keys(filter string) ([]string, error) {
var out []string
for k := range m {
if ok, _ := matchKey(filter, k); ok {
out = append(out, k)
}
}
return out, nil
}
func matchKey(filter, key string) (bool, error) {
f, k := strings.Split(filter, "."), strings.Split(key, ".")
if len(f) != len(k) {
return false, nil
}
for i := range f {
if f[i] != "*" && f[i] != k[i] {
return false, nil
}
}
return true, nil
}
type askRecorder struct{ asked []oc.Ask }
func (a *askRecorder) Ask(x oc.Ask) error { a.asked = append(a.asked, x); return nil }
const pkey = "main.5.app"
// fixture is a pipeline of app with a required lint asked of node-build-go, an optional unit asked of the same seat and an
// approval step waiting, and a hearer over it.
func fixture() (*Hearer, memStore, *[]string, *askRecorder, map[string]seat.KeptResult, map[string]string) {
store := memStore{pkey: {p: seat.Pipeline{V: 1, Pipeline: pkey, Branch: "main", Source: 5, Module: "app", Commit: "c1", Fingerprint: "src2:aa",
Outcome: seat.PipelineOutcomeRunning, Steps: []seat.StepResult{
{Artifact: "program", Step: seat.StepResultStepLint, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "a1", Seat: nodebuildgo.Seat},
{Artifact: "program", Step: seat.StepResultStepUnit, Status: seat.StepResultStatusOptional, Outcome: seat.StepResultOutcomeRunning, Ask: "a2", Seat: nodebuildgo.Seat},
{Artifact: "program", Step: seat.StepResultStepApproval, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeWaiting},
}}, rev: 1}}
said := &[]string{}
asker := &askRecorder{}
kept := map[string]seat.KeptResult{}
outs := map[string]string{}
h := &Hearer{Store: store, Name: "mesh-pipeline-runner", Asker: asker,
Say: func(subject string, _ any) error {
*said = append(*said, subject[strings.Index(subject, ".event.")+7:])
return nil
},
Log: func(seat.LogEntry) error { return nil },
Keep: func(k string, v seat.KeptResult) error { kept[k] = v; return nil },
Output: func(k string, d []byte) error { outs[k] = string(d); return nil },
Now: func() time.Time { return time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) }}
return h, store, said, asker, kept, outs
}
func ended(ask, step, outcome string) StepEnded {
return StepEnded{ID: ask, Ask: ask, Node: "n1", Pipeline: pkey, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: step, Outcome: outcome}
}
var goSeat = StepSeat{Seat: nodebuildgo.Seat, Prefix: "mesh.seat.node-build-go.event.step-ended."}
// A step's end is taken only as the runner asked it (design 47 G §24; ADR 0666 §21).
func TestStepEndedIsTakenOnlyAsAsked(t *testing.T) {
h, store, _, _, _, _ := fixture()
other := ended("a1", "lint", "passed")
other.Fingerprint = "src2:bb"
wrongNode := ended("a1", "lint", "passed")
wrongNode.Node = "n2"
unknownPipeline := ended("a1", "lint", "passed")
unknownPipeline.Pipeline = "main.9.app"
for name, c := range map[string]struct {
subject string
ev StepEnded
}{
"a subject of another seat": {"mesh.seat.node-build-node.event.step-ended.n1", ended("a1", "lint", "passed")},
"a body naming another node": {goSeat.Prefix + "n1", wrongNode},
"an unknown pipeline": {goSeat.Prefix + "n1", unknownPipeline},
"an ask nobody made": {goSeat.Prefix + "n1", ended("zz", "lint", "passed")},
"another fingerprint": {goSeat.Prefix + "n1", other},
"another step": {goSeat.Prefix + "n1", ended("a1", "unit", "passed")},
"an outcome step-ended lacks": {goSeat.Prefix + "n1", ended("a1", "lint", "waiting")},
} {
if _, err := h.OnStepEnded(goSeat, c.subject, c.ev); !errors.Is(err, ErrRefused) {
t.Errorf("%s: err %v, want a refusal", name, err)
}
}
if store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning {
t.Error("a refused end changed the pipeline")
}
// The seat the step was asked of is the only one that ends it.
if _, err := h.OnStepEnded(StepSeat{Seat: "node-build-node", Prefix: "mesh.seat.node-build-node.event.step-ended."}, "mesh.seat.node-build-node.event.step-ended.n1", ended("a1", "lint", "passed")); !errors.Is(err, ErrRefused) {
t.Errorf("another seat's holder ended the step: %v", err)
}
}
// A taken end is written, kept and said; the pipeline ends once every step ended (§24, §27).
func TestStepEndedEndsThePipeline(t *testing.T) {
h, store, said, _, kept, outs := fixture()
h.Asker = nil
lint := ended("a1", "lint", "passed")
lint.Output = "all clean"
unit := ended("a2", "unit", "failed")
unit.Kept = []nodebuildgo.KeptResult{{Digest: "sha256:d1", Level: "unit", Package: "app/x"}}
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", lint); err != nil {
t.Fatal(err)
}
if p := store[pkey].p; p.Outcome != seat.PipelineOutcomeRunning || p.Steps[0].Node != "n1" || p.Steps[0].Output == "" {
t.Fatalf("after the lint: %+v", p)
}
if len(outs) != 1 {
t.Errorf("outputs kept %v", outs)
}
// Approval waits with no asker, so the pipeline does not end on the last test step alone.
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", unit); err != nil {
t.Fatal(err)
}
if p := store[pkey].p; p.Outcome != seat.PipelineOutcomeRunning {
t.Fatalf("a pipeline with its approval waiting ended: %+v", p)
}
if k := kept[seat.KeptByLevelDigestKey("unit", keys.EscapeName("sha256:d1"))]; k.Package != "app/x" || k.Pipeline != pkey {
t.Errorf("kept %+v", kept)
}
// A step heard again takes nothing.
n := len(*said)
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", unit); err != nil || len(*said) != n {
t.Errorf("a second end said %v (%v)", (*said)[n:], err)
}
}
// The approval step is asked last, only when every other required step passed, and ended by its warrant (§26, §37).
func TestApprovalIsAskedLastAndEndedByItsWarrant(t *testing.T) {
h, store, said, asker, _, _ := fixture()
var record *oc.Warrant
h.Where = func(string) (string, *oc.Warrant, error) { return "chosen", record, nil }
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ended("a1", "lint", "passed")); err != nil || len(asker.asked) != 0 {
t.Fatalf("asked %d before the unit step ended (%v)", len(asker.asked), err)
}
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ended("a2", "unit", "failed")); err != nil {
t.Fatal(err)
}
// An optional step that failed holds nothing: the approval is asked.
if len(asker.asked) != 1 || asker.asked[0].About != "app.main" {
t.Fatalf("asked %+v", asker.asked)
}
a := asker.asked[0]
if got := (*said)[len(*said)-1]; !strings.HasPrefix(got, "wait-held.app.main.c1") {
t.Errorf("the wait was said as %q", got)
}
warrant := func(option, outcome string) oc.Warrant {
return oc.Warrant{Asker: "mesh-pipeline-runner", Ask: a.ID, About: a.About, Outcome: outcome, Option: option, Level: operatorchannelclient.Approve,
AskDigest: operatorchannelclient.Digest(a), At: operatorchannelclient.FormatTime(time.Date(2026, 11, 11, 13, 0, 0, 0, time.UTC)),
By: &oc.Person{Who: operatorchannelclient.Operator, Identity: "op", Kind: "telegram", Verified: "linked"}}
}
// The event only wakes the runner (A-89): an event that says approve while the router's record holds none takes nothing.
event := warrant(OptionApprove, oc.WarrantOutcomeChosen)
if _, err := h.OnDecided(event); !errors.Is(err, ErrNoRecord) || store[pkey].p.Steps[2].Outcome != seat.StepResultOutcomeRunning {
t.Errorf("an event with no record was taken: %v", err)
}
// A record whose warrant is for another ask's digest is not taken; an event of another asker neither.
bad := warrant(OptionApprove, oc.WarrantOutcomeChosen)
bad.AskDigest = "sha256:other"
record = &bad
if _, err := h.OnDecided(event); err == nil || store[pkey].p.Steps[2].Outcome != seat.StepResultOutcomeRunning {
t.Errorf("a warrant for another ask was taken: %v", err)
}
notMine := event
notMine.Asker = "someone"
if _, err := h.OnDecided(notMine); !errors.Is(err, ErrRefused) {
t.Errorf("another asker's event: %v", err)
}
// The record says decline although the event body says approve: the record decides.
decline := warrant(OptionDecline, oc.WarrantOutcomeChosen)
record = &decline
if p, err := h.OnDecided(event); err != nil || p.Steps[2].Outcome != seat.StepResultOutcomeDeclined {
t.Fatalf("the record said decline: %+v (%v)", p.Steps[2], err)
}
// (the declined pipeline above is a fresh fixture's: approve it on a new one)
h, store, said, asker, _, _ = fixture()
h.Where = func(string) (string, *oc.Warrant, error) { return "chosen", record, nil }
for _, s := range []StepEnded{ended("a1", "lint", "passed"), ended("a2", "unit", "failed")} {
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", s); err != nil {
t.Fatal(err)
}
}
a = asker.asked[0]
approve := warrant(OptionApprove, oc.WarrantOutcomeChosen)
record = &approve
p, err := h.OnDecided(event)
if err != nil || p.Steps[2].Outcome != seat.StepResultOutcomePassed || p.Outcome != seat.PipelineOutcomePassed || p.EndCount != 1 {
t.Fatalf("approved: %+v (%v)", p, err)
}
if !strings.HasPrefix((*said)[len(*said)-1], "pipeline-ended.app.main.c1") {
t.Errorf("said %v", *said)
}
// A pipeline that ended takes no second decision.
record = &decline
if p, err := h.OnDecided(event); err != nil || p.Steps[2].Outcome != seat.StepResultOutcomePassed {
t.Errorf("a second warrant changed it: %+v (%v)", p.Steps[2], err)
}
}
// A decline, an expiry or a replacement ends the approval step declined or superseded, and a failed required step leaves it
// not asked with the pipeline failed (§26, §37).
func TestApprovalEnds(t *testing.T) {
for name, c := range map[string]struct {
outcome, option, want string
}{
"declined": {oc.WarrantOutcomeChosen, OptionDecline, seat.StepResultOutcomeDeclined},
"expired": {oc.WarrantOutcomeExpired, "", seat.StepResultOutcomeDeclined},
"refused": {oc.WarrantOutcomeRefused, "", seat.StepResultOutcomeDeclined},
"superseded": {oc.WarrantOutcomeReplaced, "", seat.StepResultOutcomeSuperseded},
} {
h, _, _, asker, _, _ := fixture()
for _, s := range []StepEnded{ended("a1", "lint", "passed"), ended("a2", "unit", "passed")} {
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", s); err != nil {
t.Fatal(err)
}
}
a := asker.asked[0]
rec := oc.Warrant{Asker: h.Name, Ask: a.ID, About: a.About, Outcome: c.outcome, Option: c.option, Level: operatorchannelclient.Approve,
AskDigest: operatorchannelclient.Digest(a), At: operatorchannelclient.FormatTime(time.Date(2026, 11, 11, 13, 0, 0, 0, time.UTC)),
By: &oc.Person{Who: operatorchannelclient.Operator, Identity: "op", Kind: "telegram", Verified: "linked"}}
h.Where = func(string) (string, *oc.Warrant, error) { return c.outcome, &rec, nil }
p, err := h.OnDecided(oc.Warrant{Asker: h.Name, Ask: a.ID, About: a.About})
if err != nil || p.Steps[2].Outcome != c.want || p.Outcome != seat.PipelineOutcomeFailed {
t.Errorf("%s: step %q, pipeline %q (%v)", name, p.Steps[2].Outcome, p.Outcome, err)
}
}
h, store, _, asker, _, _ := fixture()
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ended("a1", "lint", "failed")); err != nil {
t.Fatal(err)
}
if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ended("a2", "unit", "passed")); err != nil {
t.Fatal(err)
}
p := store[pkey].p
if len(asker.asked) != 0 || p.Steps[2].Outcome != seat.StepResultOutcomeNotAsked || p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(p.Why, "lint") {
t.Errorf("asked %d; %+v", len(asker.asked), p)
}
}
// An optional step still waiting (a code review nobody has given) does not hold a pipeline's end; a required one does
// (design 47 G A-104).
func TestAnOptionalStepStillWaitingHoldsNoEnd(t *testing.T) {
now := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)
for name, c := range map[string]struct {
review string
ends bool
}{"optional review waiting": {seat.StepResultStatusOptional, true}, "required review waiting": {seat.StepResultStatusRequired, false}} {
p := seat.Pipeline{Outcome: seat.PipelineOutcomeRunning, Steps: []seat.StepResult{
{Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomePassed},
{Step: seat.StepResultStepReview, Status: c.review, Outcome: seat.StepResultOutcomeWaiting},
{Step: seat.StepResultStepApproval, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomePassed},
}}
if got := endPipeline(&p, now); got != c.ends {
t.Errorf("%s: ended = %v, want %v", name, got, c.ends)
}
if c.ends && p.Outcome != seat.PipelineOutcomePassed {
t.Errorf("%s: outcome %s", name, p.Outcome)
}
}
}
// An ask no holder took within its bound ends its step `could not run` naming the seat, and with the rest ended the pipeline
// failed (design 47 G §32, §535, §647, A-104); a step that ended, or ask nobody placed, ends nothing.
func TestAnAskNoHolderTookEndsItsStepNamingTheSeat(t *testing.T) {
h, store, said, _, _, _ := fixture()
e := store[pkey]
e.p.Steps[1].Outcome = seat.StepResultOutcomePassed // the unit step ended; only lint is waiting for a holder
e.p.Steps[2].Outcome = seat.StepResultOutcomeNotAsked
store[pkey] = e
if err := h.NotTaken("nobody-asked", AskBound); err != nil || store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning {
t.Fatalf("an ask no step names ended something: %v", err)
}
if err := h.NotTaken("a1", AskBound); err != nil {
t.Fatal(err)
}
p := store[pkey].p
if p.Steps[0].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(p.Steps[0].Why, nodebuildgo.Seat) ||
p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(strings.Join(*said, " "), seat.EventPipelineEnded) {
t.Errorf("step %+v, pipeline %s (%s), said %v: want could not run naming %s and a failed pipeline", p.Steps[0], p.Outcome, p.Why, *said, nodebuildgo.Seat)
}
// Its end taken once: the bound's expiry heard again changes nothing.
n := len(*said)
if err := h.NotTaken("a1", AskBound); err != nil || len(*said) != n {
t.Errorf("a second expiry said %d more events: %v", len(*said)-n, err)
}
}
// The bound is armed before the ask is put, only for the build seats whose holder says ask-taken, and an ask whose bound is
// not armed is not put (§535, §647).
func TestTheBoundIsArmedBeforeAnAskIsPut(t *testing.T) {
var order []string
asker := askerFunc(func(accept, id string, _ any) error { order = append(order, "put "+id); return nil })
arm := Arm(func(ask string) error { order = append(order, "arm "+ask); return nil })
asks := []Asked{{Seat: nodebuildgo.Seat, ID: "step-1", Ask: 1}, {Seat: wants.Seat, ID: "step-2", Ask: 2}}
if err := askAll(asker, arm, "k", asks); err != nil || strings.Join(order, ",") != "arm step-1,put step-1,put step-2" {
t.Errorf("order %v, %v; want the build seat's bound armed first and mesh-wants' ask with none", order, err)
}
order = nil
failing := Arm(func(string) error { return errors.New("bucket gone") })
if err := askAll(asker, failing, "k", asks[:1]); err == nil || len(order) != 0 {
t.Errorf("an ask whose bound cannot be armed was put or said nothing: %v %v", order, err)
}
}
type askerFunc func(accept, id string, body any) error
func (f askerFunc) AskStep(accept, id string, body any) error { return f(accept, id, body) }
@@ -0,0 +1,28 @@
package pipeline
import (
"encoding/json"
"strings"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
)
// ToolchainOf reads a toolchain-agreed as the toolchain a step ask names: its language and version (design 47 G §416, A-104).
// The recipe and the building node's digest are the seat's to report, not the runner's to know.
func ToolchainOf(raw []byte) (*nodebuildgo.ToolchainUsed, error) {
var t builds.ToolchainAgreed
if err := json.Unmarshal(raw, &t); err != nil {
return nil, err
}
if t.Language == "" {
return nil, nil
}
return &nodebuildgo.ToolchainUsed{Language: t.Language, Version: t.Version}, nil
}
// InPackage says whether a kept result's package is the artifact's package name or lies beneath it (A-104: the kept entries
// whose package is the artifact's package, a Go module's packages being the module path and the paths beneath it).
func InPackage(pkg, name string) bool {
return name != "" && (pkg == name || strings.HasPrefix(pkg, name+"/"))
}
@@ -0,0 +1,336 @@
package pipeline
import (
"errors"
"fmt"
"strings"
"time"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/views"
)
// Verbs are the runner's write verbs `review`, `rerun`, `accept` and `cancel`, answered by its own process (design 47 G §22,
// §70, §482, A-107): each writes the state pipelines by compare-and-set on the revision read and says its event as the
// hearer does for a step's end. The caller each takes is the one the bus proved.
type Verbs struct {
Hearer *Hearer
// Starter asks a step again (rerun), as it asks the steps of a start; View reads a branch's catalog half for the ask.
Starter *Starter
View func(branch string) (Half, bool)
// Acts records a hand act before the verb does it (rerun, accept, cancel); a nil Acts refuses the verb, nothing recorded.
Acts views.HandActs
// Drop drops the bound of an ask a cancelled pipeline no longer waits on.
Drop func(ask string) error
}
func (v Verbs) now() time.Time { return v.Hearer.now() }
func (v Verbs) record(verb, what, why string) error {
if v.Acts == nil {
return errors.New("no way to record a hand act is given: nothing was done")
}
if strings.TrimSpace(why) == "" {
return fmt.Errorf("%s says why: nothing was done", verb)
}
if err := v.Acts.RecordHandAct(views.HandAct{What: what, Why: why, Verb: seat.Seat + "." + verb}); err != nil {
return fmt.Errorf("the hand act is not recorded, so nothing was done: %w", err)
}
return nil
}
// change writes one change of a pipeline by compare-and-set, read again when another write came between, and says what it
// became: the step-ended of the step mutate changed (its index, -1 for none), the wait of an approval asked and the
// pipeline-ended of a pipeline it ended. It returns the written pipeline and the changed step.
func (v Verbs) change(key, event string, record func() error, mutate func(p *seat.Pipeline, now time.Time) (int, error)) (seat.Pipeline, int, error) {
h := v.Hearer
recorded := false
for {
p, rev, err := h.Store.Get(key)
if err != nil {
return p, -1, fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
now := h.now()
i, err := mutate(&p, now)
if err != nil {
return p, -1, err
}
// The hand act is recorded once the verb is valid and before it writes (ADR 0565 clause 11); a refused verb records none.
if record != nil && !recorded {
if err := record(); err != nil {
return p, -1, err
}
recorded = true
}
asked, err := askApproval(key, &p, now, h.Asker)
if err != nil {
return p, i, err
}
ended := endPipeline(&p, now)
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return p, i, fmt.Errorf("pipeline %s cannot be written: %w", key, err)
}
if i >= 0 {
if event == seat.EventStepEnded {
h.logEnd(key, p.Steps[i], now)
}
if err := h.say(event, p, &p.Steps[i]); err != nil {
return p, i, fmt.Errorf("pipeline %s's %s is written but not said: %w", key, event, err)
}
}
return p, i, h.said(p, asked, ended)
}
}
// Review takes a code review's result for a commit and fingerprint from caller, the bus's proof, refusing none for
// authorship (§482): it ends the placed review step of each running pipeline of the module at that commit and fingerprint,
// recording caller in the step's result. It answers the first such pipeline; a review no pipeline waits on is refused.
func (v Verbs) Review(caller string, in seat.ReviewInput) (seat.ReviewAnswer, error) {
if in.Outcome != seat.ReviewInputOutcomePassed && in.Outcome != seat.ReviewInputOutcomeFailed {
return seat.ReviewAnswer{}, fmt.Errorf("a review ends %q or %q, not %q", seat.ReviewInputOutcomePassed, seat.ReviewInputOutcomeFailed, in.Outcome)
}
keys, err := v.Hearer.Store.Keys("*.*.*")
if err != nil {
return seat.ReviewAnswer{}, fmt.Errorf("the pipelines are not listed: %w", err)
}
var answer *seat.ReviewAnswer
for _, key := range keys {
p, _, err := v.Hearer.Store.Get(key)
if err != nil || p.Module != in.Module || p.Commit != in.Commit || p.Fingerprint != in.Fingerprint || reviewWaiting(p) < 0 {
continue
}
written, i, err := v.change(key, seat.EventStepEnded, nil, func(p *seat.Pipeline, now time.Time) (int, error) {
i := reviewWaiting(*p)
if i < 0 {
return -1, fmt.Errorf("%s has no review step waiting any more", key)
}
s := &p.Steps[i]
s.Outcome, s.By, s.Why, s.Ended = in.Outcome, caller, in.Said, &now
return i, nil
})
if err != nil {
return seat.ReviewAnswer{}, err
}
if answer == nil {
answer = &seat.ReviewAnswer{Pipeline: key, Result: written.Steps[i]}
}
}
if answer == nil {
return seat.ReviewAnswer{}, fmt.Errorf("no running pipeline of %s at %s (%s) waits on a review", in.Module, in.Commit, in.Fingerprint)
}
return *answer, nil
}
func reviewWaiting(p seat.Pipeline) int {
if p.Outcome != seat.PipelineOutcomeRunning {
return -1
}
for i, s := range p.Steps {
if s.Step == seat.StepResultStepReview && s.Outcome == seat.StepResultOutcomeWaiting {
return i
}
}
return -1
}
// stepOf is the index of the step named, and of its artifact when one is named.
func stepOf(p seat.Pipeline, step, artifact string) (int, error) {
found := -1
for i, s := range p.Steps {
if s.Step != step || (artifact != "" && s.Artifact != artifact) {
continue
}
if found >= 0 {
return -1, fmt.Errorf("%s names more than one %s step: name its artifact", p.Pipeline, step)
}
found = i
}
if found < 0 {
return -1, fmt.Errorf("%s has no %s step", p.Pipeline, step)
}
return found, nil
}
// Accept accepts a failed `optional` step, recorded in the hand-act log first with who and why (§30, §394).
func (v Verbs) Accept(caller string, in seat.AcceptInput) (seat.AcceptAnswer, error) {
p, i, err := v.change(in.Pipeline, seat.EventStepEnded, func() error {
return v.record(seat.VerbAccept, fmt.Sprintf("accepted the failed %s step of %s", in.Step, in.Pipeline), in.Why)
}, func(p *seat.Pipeline, now time.Time) (int, error) {
i, err := stepOf(*p, in.Step, in.Artifact)
if err != nil {
return -1, err
}
s := &p.Steps[i]
if s.Status != seat.StepResultStatusOptional || s.Outcome != seat.StepResultOutcomeFailed {
return -1, fmt.Errorf("only a failed optional step is accepted: the %s step of %s is %s and %s", in.Step, in.Pipeline, s.Status, s.Outcome)
}
s.Outcome, s.Accepted, s.Ended = seat.StepResultOutcomeAccepted, &seat.Accepted{At: &now, By: caller, Why: in.Why}, &now
return i, nil
})
if err != nil {
return seat.AcceptAnswer{}, err
}
return seat.AcceptAnswer{Result: p.Steps[i]}, nil
}
// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and the bound of each ask dropped, recorded in
// the hand-act log first (§394); it says pipeline-ended.
func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, error) {
h := v.Hearer
recorded := false
for {
p, rev, err := h.Store.Get(in.Pipeline)
if err != nil {
return seat.CancelAnswer{}, fmt.Errorf("the pipeline %s is not read: %w", in.Pipeline, err)
}
if p.Outcome != seat.PipelineOutcomeRunning {
return seat.CancelAnswer{}, fmt.Errorf("the pipeline %s is %s, not running", in.Pipeline, p.Outcome)
}
if !recorded {
if err := v.record(seat.VerbCancel, "cancelled the pipeline "+in.Pipeline, in.Why); err != nil {
return seat.CancelAnswer{}, err
}
recorded = true
}
now := h.now()
var drop []string
for i := range p.Steps {
s := &p.Steps[i]
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if s.Ask != "" {
drop = append(drop, s.Ask)
}
s.Outcome, s.By, s.Why, s.Ended = seat.StepResultOutcomeCancelled, caller, in.Why, &now
}
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeCancelled, in.Why, &now
p.EndCount++
if _, err := h.Store.Update(in.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err)
}
if v.Drop != nil {
for _, ask := range drop {
if err := v.Drop(ask); err != nil {
h.logf("the bound of ask %s of the cancelled %s is not dropped: %v", ask, in.Pipeline, err)
}
}
}
return seat.CancelAnswer{Pipeline: p}, h.said(p, false, true)
}
}
// Rerun asks one step again, recorded in the hand-act log first (§22, §39): the step goes back to running under a new ask,
// the ask saying it is a rerun and the caller the bus proved; a failed pipeline runs again and ends once more with its end
// count one higher (§325), an approval never asked waiting again. A review waits again; an approval is asked again by
// `ask-again`.
func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error) {
if in.Step == seat.StepResultStepApproval {
return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun")
}
var ask *Asked
p, _, err := v.change(in.Pipeline, seat.EventStepStarted, func() error {
return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why)
}, func(p *seat.Pipeline, now time.Time) (int, error) {
ask = nil
switch p.Outcome {
case seat.PipelineOutcomeCancelled, seat.PipelineOutcomeSuperseded:
return -1, fmt.Errorf("the pipeline %s is %s: a step of it is not asked again", p.Pipeline, p.Outcome)
}
i, err := stepOf(*p, in.Step, in.Artifact)
if err != nil {
return -1, err
}
s := &p.Steps[i]
if s.Outcome == seat.StepResultOutcomeRunning {
return -1, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, p.Pipeline)
}
if s.Seat != "" {
if ask, err = v.askOf(*p, *s, caller, now); err != nil {
return -1, err
}
}
*s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
Outcome: seat.StepResultOutcomeWaiting, By: caller, Rerun: true}
if ask != nil {
s.Outcome, s.Started, s.Ask = seat.StepResultOutcomeRunning, &now, ask.ID
}
// The pipeline runs again; an approval never asked waits again, to be asked when the step passes.
if p.Outcome != seat.PipelineOutcomeRunning {
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
}
for j := range p.Steps {
if p.Steps[j].Step == seat.StepResultStepApproval && p.Steps[j].Outcome == seat.StepResultOutcomeNotAsked {
p.Steps[j].Outcome, p.Steps[j].Why = seat.StepResultOutcomeWaiting, ""
}
}
if ask == nil {
return -1, nil
}
return i, nil
})
if err != nil {
return seat.RerunAnswer{}, err
}
if ask != nil {
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
}
}
res, err := stepOf(p, in.Step, in.Artifact)
if err != nil {
return seat.RerunAnswer{}, err
}
r := p.Steps[res]
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &r}, nil
}
// askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice
// under one id is kept once), saying it is a rerun and by whom.
func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) {
if v.Starter == nil || v.Starter.Registration == nil || v.Starter.Steps == nil || v.View == nil {
return nil, errors.New("the runner cannot ask a step: it holds no registration or no ask")
}
reg, ok := v.Starter.Registration(p.Module)
if !ok {
return nil, fmt.Errorf("%s has no registration", p.Module)
}
half, ok := v.View(p.Branch)
if !ok {
return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch)
}
_, placed := Place(p, reg, half.Entries, now, v.Starter.Name)
base := AskID(p.Pipeline, s.Artifact, s.Step)
for _, a := range placed {
if a.ID != base {
continue
}
a.ID = AskID(p.Pipeline, s.Artifact, s.Step+"\nrerun\n"+now.Format(time.RFC3339Nano))
switch b := a.Ask.(type) {
case nodebuildgo.StepAsk:
b.ID = a.ID
if b.Step != nil {
step := *b.Step
step.Rerun, step.By = true, caller
b.Step = &step
}
a.Ask = b
case builds.StepRun:
b.Rerun, b.By = true, caller
a.Ask = b
case wants.StepRun:
b.Rerun, b.By = true, caller
a.Ask = b
}
return &a, nil
}
return nil, fmt.Errorf("the %s step of %s is placed no ask now", s.Step, p.Pipeline)
}
@@ -0,0 +1,172 @@
package pipeline
import (
"errors"
"strings"
"testing"
"time"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/views"
)
type actLog []string
func (a *actLog) RecordHandAct(h views.HandAct) error {
*a = append(*a, h.Verb+": "+h.What+" / "+h.Why)
return nil
}
// verbsFixture is a running pipeline of app placed from a policy with a required lint, an optional unit, an optional review
// and a required approval, in a store, with the verbs over it.
func verbsFixture() (Verbs, memStore, *[]string, *stepAsks, *actLog, *[]string) {
reg, entries := placeFixture()
reg.Policy.Steps.Review = "optional"
h, store, said, _, _, _ := fixture()
p := seat.Pipeline{Pipeline: pkey, Branch: "main", Source: 5, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app",
Outcome: seat.PipelineOutcomeRunning}
p.Steps, _ = Place(p, reg, entries, time.Unix(1, 0), "the-runner")
store[pkey] = struct {
p seat.Pipeline
rev uint64
}{p, 1}
asks, acts, dropped := &stepAsks{}, &actLog{}, &[]string{}
s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true },
Now: h.Now}
return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts,
Drop: func(ask string) error { *dropped = append(*dropped, ask); return nil }}, store, said, asks, acts, dropped
}
func stepNamed(p seat.Pipeline, artifact, step string) seat.StepResult {
for _, s := range p.Steps {
if s.Artifact == artifact && s.Step == step {
return s
}
}
return seat.StepResult{}
}
// A code review ends the placed review step of the pipeline at its commit and fingerprint, whoever its author, recording the
// caller the bus proved in the step's result, its step-ended and the pipeline's show; a review no pipeline waits on is refused
// (design 47 G §70, §482).
func TestReviewEndsThePlacedReviewStepWithItsProvenCaller(t *testing.T) {
v, store, said, _, _, _ := verbsFixture()
in := seat.ReviewInput{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Outcome: "passed", Said: "reads well"}
for name, bad := range map[string]seat.ReviewInput{
"another commit": {Module: "app", Commit: "c2", Fingerprint: "src2:aa", Outcome: "passed"},
"another fingerprint": {Module: "app", Commit: "c1", Fingerprint: "src2:bb", Outcome: "passed"},
"an outcome review lacks": {Module: "app", Commit: "c1", Fingerprint: "src2:aa", Outcome: "waiting"},
} {
if _, err := v.Review("agent.dev", bad); err == nil {
t.Errorf("%s: a review was taken", name)
}
}
a, err := v.Review("agent.dev", in)
if err != nil || a.Pipeline != pkey || a.Result.Outcome != "passed" || a.Result.By != "agent.dev" {
t.Fatalf("review = %+v, %v", a, err)
}
got := stepNamed(store[pkey].p, "", seat.StepResultStepReview)
if got.Outcome != "passed" || got.By != "agent.dev" || got.Why != "reads well" || got.Ended == nil {
t.Errorf("the review step = %+v", got)
}
if len(*said) != 1 || !strings.HasPrefix((*said)[0], "step-ended.") {
t.Errorf("said %v, want one step-ended", *said)
}
if _, err := v.Review("agent.dev", in); err == nil {
t.Error("a second review of an ended review step was taken")
}
}
// accept takes only a failed optional step, writes its hand act first, and refuses without one; rerun asks the step again
// under a new ask saying it is a rerun and its caller, reopens a failed pipeline and asks nothing of a refused one; cancel ends
// a running pipeline cancelled, each step cancelled and each ask's bound dropped (design 47 G §22, §30, §325, §394).
func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testing.T) {
v, store, said, asks, acts, dropped := verbsFixture()
end := func(step, outcome string) {
p, rev, _ := store.Get(pkey)
for i := range p.Steps {
if p.Steps[i].Artifact == "app:program" && p.Steps[i].Step == step {
p.Steps[i].Outcome = outcome
}
}
store.Update(pkey, p, rev)
}
end("unit", seat.StepResultOutcomeFailed)
if _, err := v.Accept("operator", seat.AcceptInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "x"}); err == nil {
t.Error("a running required step was accepted")
}
if _, err := v.Accept("operator", seat.AcceptInput{Pipeline: pkey, Step: "unit", Artifact: "app:program"}); err == nil {
t.Error("an accept with no why was taken")
}
a, err := v.Accept("operator", seat.AcceptInput{Pipeline: pkey, Step: "unit", Artifact: "app:program", Why: "flaky"})
if err != nil || a.Result.Outcome != "accepted" || a.Result.Accepted == nil || a.Result.Accepted.By != "operator" || a.Result.Accepted.Why != "flaky" {
t.Fatalf("accept = %+v, %v", a, err)
}
if len(*acts) != 1 || !strings.Contains((*acts)[0], "mesh-pipeline-runner.accept") || !strings.Contains((*acts)[0], "flaky") {
t.Errorf("hand acts = %v", *acts)
}
// A rerun of the lint step: a failed pipeline runs again.
end("lint", seat.StepResultOutcomeFailed)
p, rev, _ := store.Get(pkey)
p.Outcome = seat.PipelineOutcomeFailed
store.Update(pkey, p, rev)
old := stepNamed(p, "app:program", "lint").Ask
r, err := v.Rerun("seat:mesh-healers", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "retry"})
if err != nil || r.Result == nil || r.Result.Outcome != "running" || r.Result.Ask == old || !r.Result.Rerun || r.Result.By != "seat:mesh-healers" {
t.Fatalf("rerun = %+v, %v", r.Result, err)
}
if len(asks.asks) != 1 || asks.ids[0] != r.Result.Ask {
t.Fatalf("asked %v under %v, want one ask under %s", asks.asks, asks.ids, r.Result.Ask)
}
if b, ok := asks.asks[0].(nodebuildgo.StepAsk); !ok || b.ID != r.Result.Ask || b.Step == nil || !b.Step.Rerun || b.Step.By != "seat:mesh-healers" || b.Step.Step != "lint" {
t.Errorf("the ask = %+v", asks.asks[0])
}
if got, _, _ := store.Get(pkey); got.Outcome != seat.PipelineOutcomeRunning {
t.Errorf("the failed pipeline is %s, want running again", got.Outcome)
}
if _, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "again"}); err == nil {
t.Error("a running step was asked again")
}
if _, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "approval", Why: "again"}); err == nil {
t.Error("an approval was asked again by rerun")
}
// A cancel ends the pipeline cancelled and drops the bounds of the asks still held.
c, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "wrong branch"})
if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) == 0 {
t.Fatalf("cancel = %+v, %v, dropped %v", c.Pipeline, err, *dropped)
}
if s := stepNamed(c.Pipeline, "app:program", "lint"); s.Outcome != "cancelled" || s.By != "operator" {
t.Errorf("the lint step = %+v", s)
}
if _, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "again"}); err == nil {
t.Error("a cancelled pipeline was cancelled again")
}
if _, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "again"}); err == nil {
t.Error("a step of a cancelled pipeline was asked again")
}
last := (*said)[len(*said)-1]
if !strings.HasPrefix(last, "pipeline-ended.") {
t.Errorf("last said %q, want pipeline-ended", last)
}
if n := len(*acts); n != 3 {
t.Errorf("%d hand acts recorded, want accept, rerun, cancel (3): %v", n, *acts)
}
}
// A hand act the log does not take is a verb that does nothing (ADR 0565 clause 11).
func TestAVerbWhoseHandActIsNotRecordedDoesNothing(t *testing.T) {
v, store, _, _, _, _ := verbsFixture()
v.Acts = failingActs{}
if _, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "x"}); err == nil {
t.Fatal("cancel did something")
}
if p, _, _ := store.Get(pkey); p.Outcome != seat.PipelineOutcomeRunning {
t.Errorf("the pipeline is %s", p.Outcome)
}
}
type failingActs struct{}
func (failingActs) RecordHandAct(views.HandAct) error { return errors.New("no holder") }
@@ -1,6 +1,7 @@
package pipeline_test
import (
"encoding/json"
"os"
"slices"
"testing"
@@ -20,8 +21,10 @@ func TestManifest(t *testing.T) {
t.Fatalf("the manifest does not read: %v", err)
}
var states, streams, objects []string
keyTTL := map[string]bool{}
for _, s := range m.State {
states = append(states, s.Name)
keyTTL[s.Name] = s.KeyTTL
}
for _, s := range m.Streams {
streams = append(streams, s.Name)
@@ -47,7 +50,9 @@ func TestManifest(t *testing.T) {
{"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat &&
equalSets(m.Claims[0].Serves, seat.Verbs())},
// §27, §85, §330, §355, §77: the old state sections.
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "findings", "first-views"})},
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "bounds", "toolchains", "findings", "first-views"})},
// §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live.
{"bounds is key-ttl", keyTTL["bounds"]},
// §27, §85: the step log on an old stream, its outputs in an old objects section.
// RN24 (§383, §387): beside it, the finding stream its findings are said on.
{"old streams section", equalSets(streams, []string{seat.StateStepLog, "mesh-delivery-pipeline-findings"})},
@@ -60,15 +65,21 @@ func TestManifest(t *testing.T) {
{"step-outputs 1 GiB", len(m.Objects) == 1 && m.Objects[0].MaxMiB == 1024},
// §333, §54, §44, §53, §376, §24, §37, §4, §77: what the runner hears.
{"consumes", equalSets(m.Consumes, []string{
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed",
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
"node-build-image.step-ended", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked"})},
// RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and
// asked again by hand (a hand act).
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act",
// RN3 (§53, §334): a publication's dependents behind.
"seat:mesh-catalog.outdated"})},
"seat:mesh-catalog.outdated",
// A-74, A-89: who holds the seat operator-channel, whose state asks holds the router's record of the approval ask.
"seat:mesh-module-assigner.assignments",
// A-102 (§22, §24): the accepted kinds the runner puts its steps on.
"seat:mesh-builds.step", "seat:mesh-wants.step", "seat:node-build-go.check.step", "seat:node-build-node.check.step",
"seat:node-build-image.check.step"})},
{"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})},
{"first-view tools", equalSets(m.Tools, []string{"pipeline_first_views", "pipeline_first_view_again"})},
// RN1/AC5 (§95, §130): the new assigner is delivered before the runner.
{"program needs the assigner's program", slices.Equal(program.Needs, []string{"mesh-delivery-assigner:program"})},
@@ -80,6 +91,31 @@ func TestManifest(t *testing.T) {
}
}
// §610, A-75: no verb of the runner's definition names its callers, so the class's default holds for each.
func TestNoVerbNamesItsCallers(t *testing.T) {
raw, err := os.ReadFile("../../contracts/seats/mesh-pipeline-runner/seat.json")
if err != nil {
t.Fatal(err)
}
var def struct {
Verbs []struct {
Name string `json:"name"`
Callers []string `json:"callers"`
} `json:"verbs"`
}
if err := json.Unmarshal(raw, &def); err != nil {
t.Fatal(err)
}
if len(def.Verbs) == 0 {
t.Fatal("the definition holds no verb")
}
for _, v := range def.Verbs {
if len(v.Callers) != 0 {
t.Errorf("the verb %s names callers %v; from G no verb does (§610)", v.Name, v.Callers)
}
}
}
func equalSets(a, b []string) bool {
x, y := slices.Clone(a), slices.Clone(b)
slices.Sort(x)
+23 -1
View File
@@ -30,10 +30,14 @@
"mesh-delivery-indexer.take-in.taken",
"mesh-builds.package-published",
"mesh-builds.toolchain-agreed",
"mesh-builds.step-ended",
"operator-channel.decided",
"node-build-go.step-ended",
"node-build-node.step-ended",
"node-build-image.step-ended",
"node-build-go.ask-taken",
"node-build-node.ask-taken",
"node-build-image.ask-taken",
"mesh-wants.step-ended",
"mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision",
@@ -43,7 +47,16 @@
"seat:mesh-catalog.registrations",
"seat:mesh-catalog.revision",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-catalog.outdated"
"seat:mesh-catalog.outdated",
"seat:mesh-module-assigner.assignments",
"seat:mesh-builds.step",
"seat:mesh-wants.step",
"seat:node-build-go.check.step",
"seat:node-build-node.check.step",
"seat:node-build-image.check.step"
],
"uses": [
"operator-channel"
],
"state": [
{
@@ -58,6 +71,15 @@
"name": "aged",
"history": 1
},
{
"name": "toolchains",
"history": 1
},
{
"name": "bounds",
"history": 1,
"key-ttl": true
},
{
"name": "findings",
"history": 1