Merge branch 'feat/47-G-build' into feat/47-G-build-assigner

This commit is contained in:
2026-11-11 12:37:06 +01:00
12 changed files with 795 additions and 9 deletions
@@ -7,6 +7,10 @@ import (
"encoding/json"
"fmt"
"os"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-sdk/go/views"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
@@ -82,9 +86,22 @@ func Tools(r pipeline.Reads) []stdio.Tool {
return out
}
// Module and Slug are the runner's module and its tools' prefix, as its manifest names them.
const (
Module = "mesh-delivery-pipeline"
Slug = "pipeline"
)
// FirstViewTools are the runner's `<module>_first_views` and `<module>_first_view_again` over its state first-views
// (design 47 G §77, §200): each view it holds or waits for, since when, and asked again by hand, a hand act.
func FirstViewTools(st stdio.KeptState, acts views.HandActs, now func() time.Time) []stdio.Tool {
return views.StatusTools(Module, Slug+"_first_views", Slug+"_first_view_again", st, acts, now)
}
func main() {
r := pipeline.Reads{Pipelines: kept{stdio.State(seat.StatePipelines)}, Kept: kept{stdio.State(seat.StateKept)}}
if err := stdio.Serve("", Tools(r)); err != nil {
tools := append(Tools(r), FirstViewTools(stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
os.Exit(1)
}
@@ -5,6 +5,9 @@ import (
"strings"
"testing"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-pipeline/internal/pipeline"
)
@@ -13,13 +16,17 @@ type none struct{}
func (none) Keys() ([]string, error) { return nil, nil }
func (none) Value(string) (json.RawMessage, error) { return nil, nil }
// TestTools: the bundle serves the five read verbs as <seat>.<verb> (§83, §306; RN15/AC1).
// TestTools: the bundle serves the five read verbs as <seat>.<verb> (§83, §306; RN15/AC1), and the first-view tools
// (§77, §200; RN2/AC4).
func TestTools(t *testing.T) {
var names []string
for _, tl := range Tools(pipeline.Reads{Pipelines: none{}, Kept: none{}}) {
names = append(names, tl.Name)
}
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept"
for _, tl := range FirstViewTools(stdio.State(views.StatusState), nil, nil) {
names = append(names, tl.Name)
}
want := "mesh-pipeline-runner.show mesh-pipeline-runner.list mesh-pipeline-runner.stands mesh-pipeline-runner.why-carried mesh-pipeline-runner.kept pipeline_first_views pipeline_first_view_again"
if got := strings.Join(names, " "); got != want {
t.Errorf("tools %q, want %q", got, want)
}
@@ -5,6 +5,7 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
@@ -12,8 +13,12 @@ import (
"os/signal"
"syscall"
"github.com/nats-io/nats.go"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
func main() {
@@ -39,9 +44,39 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
defer conn.Close()
// The state pipelines, `<module>_pipelines`, holds every pipeline by compare-and-set (§27): without it the
// runner keeps nothing, so it stops.
if _, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline"); err != nil {
pipelines, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline")
if err != nil {
return err
}
// The registrations and each watched branch's catalog half, first views then events (§21, §195, §200): until a
// branch's view is held the runner acts on nothing new of it, and says so.
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
}
defer calls.Close()
hub := process.NewEventHub(logger.Printf)
rv, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf)
if err != nil {
return err
}
// A pipeline starts by compare-and-set on its key's absence and says pipeline-started (§54, §103); a publish starts
// one per dependent behind (§53, §334).
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf,
Say: func(subject string, body any) error {
data, err := json.Marshal(body)
if err == nil {
_, err = js.Publish(ctx, subject, data)
}
return err
}}
if err := hearPublished(ctx, hub, starter, calls, rv.Catalog, logger.Printf); err != nil {
return err
}
if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil {
logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err)
}
go func() { <-ctx.Done(); hub.Stop() }()
logger.Printf("holding %s", seat.Seat)
<-ctx.Done()
return nil
@@ -0,0 +1,71 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// outdatedVerb is the catalog's verb a publication's dependents behind are asked by, as the manifest's invokes names it.
const outdatedVerb = "seat:" + catalog.Seat + "." + catalog.VerbOutdated
// createdKV is the state pipelines written by compare-and-set on a key's absence (§54).
type createdKV struct {
ctx context.Context
kv jetstream.KeyValue
}
func (c createdKV) Create(key string, value []byte) error {
_, err := c.kv.Create(c.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return pipeline.ErrExists
}
return err
}
// hearPublished starts the pipelines of each package-published heard (§53, §334): the dependents the catalog's outdated
// names behind, each asked as a kept call. The message is acknowledged once its starts are written, on a goroutine of its
// own, so a slow catalog holds no other event.
func hearPublished(ctx context.Context, hub *process.EventHub, starter *pipeline.Starter, calls *process.Conn, v *pipeline.Views, logf func(string, ...any)) error {
outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
raw, err := calls.CallKept(outdatedVerb, in)
if err != nil {
return catalog.OutdatedAnswer{}, err
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return catalog.OutdatedAnswer{}, fmt.Errorf("%s answered: %s", outdatedVerb, said)
}
var a catalog.OutdatedAnswer
err = json.Unmarshal(answer, &a)
return a, err
}
return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) {
var ev builds.PackagePublished
if err := json.Unmarshal(m.Data(), &ev); err != nil {
logf("%s is no package-published: %v", m.Subject(), err)
_ = m.Ack()
return
}
go func() {
n, err := starter.Published(ev, outdated, v.View)
if err != nil {
logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err)
_ = m.Nak()
return
}
logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n)
_ = m.Ack()
}()
})
}
@@ -0,0 +1,140 @@
package main
import (
"context"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-pipeline/internal/pipeline"
)
// catalogRevisions is every watched branch's catalog-revision, `mesh.seat.mesh-catalog.event.catalog-revision.<branch>`.
const catalogRevisions = "mesh.seat." + catalog.Seat + ".event." + catalog.EventCatalogRevision + ".>"
// runnerViews are the runner's first views (design 47 G §21, §195, §200): the registrations, then the catalog half of
// each watched branch they name, each said in `<module>_first_views` (its state first-views) and, past five minutes
// without a view, as view-waits in its findings.
type runnerViews struct {
Registrations *meshcatalogclient.Registrations
Catalog *pipeline.Views
}
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
// after, then start hub once.
func startViews(ctx context.Context, js jetstream.JetStream, conn process.Subscriber, cred process.Account, hub *process.EventHub,
call meshcatalogclient.Call, logf func(string, ...any)) (*runnerViews, error) {
findings := process.AwaitBucket(ctx, js, conn, mcc.FindingsBucket(cred.Module), "a view waited for past five minutes is said in the log alone", logf)
status := process.AwaitBucket(ctx, js, conn, cred.Module+"_"+views.StatusState, "the runner's first views are not said in its first-view tool, nor asked again by hand", logf)
writer := func() views.FindingWriter { return sdkfindings.Writer(ctx, findings.KV) }
reader := &views.ProcessReader{Module: cred.Module, Call: call, Logf: logf, Hub: hub, Findings: writer}
// A watched branch first named by a registration gets its view of the catalog half (CT10), its conditions and its
// status kept as every first view's, asked at once.
wire := func(f *views.Follower) {
f.Logf = logf
f.Raise = func(s views.Status) {
putFinding(writer(), views.UnansweredFinding(cred.Module, s, time.Now()), true, logf)
}
f.Clear = func(s views.Status) {
putFinding(writer(), views.UnansweredFinding(cred.Module, s, time.Now()), false, logf)
}
f.RaiseWaits = func(s views.Status) {
logf("waits for %s's view %s since %s", s.Owner, s.WaitsFor, s.WaitingSince.Format(time.RFC3339))
putFinding(writer(), views.ViewWaitsFinding(cred.Module, s, time.Now()), true, logf)
}
f.ClearWaits = func(s views.Status) {
putFinding(writer(), views.ViewWaitsFinding(cred.Module, s, time.Now()), false, logf)
}
f.Said = func(s views.Status) {
if kv := status.KV(); kv != nil {
views.KeepStatus(ctx, kv, logf)(s)
}
}
status.Ready(func(kv jetstream.KeyValue) {
if err := views.HearAgain(ctx, kv, []*views.Follower{f}, logf); err != nil {
logf("the hand asks of the first view %s are not heard (%v) until the runner starts again", f.Tool, err)
}
})
}
var (
mu sync.Mutex
made []*views.Follower
)
cv := &pipeline.Views{Logf: logf, Open: func(branch string) (pipeline.BranchView, error) {
c, err := meshcatalogclient.NewCatalog(call, branch)
if err != nil {
return nil, err
}
wire(c.Follower)
mu.Lock()
made = append(made, c.Follower)
mu.Unlock()
go func() {
if err := c.Start(ctx); err != nil {
logf("the first view of the catalog half of %s was not answered: %v", branch, err)
}
}()
return catalogView{c}, nil
}}
regs, _, err := meshcatalogclient.WireRegistrationsProcess(reader, call, func(reg catalog.Registration, gone bool) {
if !gone {
cv.Registered(reg)
}
})
if err != nil {
return nil, err
}
if err := hub.Handle(catalogRevisions, func(m jetstream.Msg) {
if err := cv.HearRevision(ctx, m.Data()); err != nil {
logf("%s is not taken: %v", m.Subject(), err)
}
_ = m.Ack()
}); err != nil {
return nil, err
}
// A consumer made after a view was asked does not deliver what was said before it: each branch's view asks again.
hub.OnBound(func(created time.Time) {
mu.Lock()
fs := append([]*views.Follower(nil), made...)
mu.Unlock()
for _, f := range fs {
f.ConsumerMade(ctx, created)
}
})
reader.Start(ctx, js, cred, status)
return &runnerViews{Registrations: regs, Catalog: cv}, nil
}
func putFinding(w views.FindingWriter, f views.Finding, raised bool, logf func(string, ...any)) {
var err error
if raised {
err = w.Put(f)
} else {
err = w.Delete(f)
}
if err != nil {
logf("the finding %s is not written: %v", f.Kind, err)
}
}
// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it.
type catalogView struct{ c *meshcatalogclient.Catalog }
func (v catalogView) Hear(ctx context.Context, data []byte) error { return v.c.Hear(ctx, data) }
func (v catalogView) View() (pipeline.Half, bool) {
h, ok := v.c.View()
if !ok {
return pipeline.Half{}, false
}
return pipeline.Half{Revision: h.Revision.Revision, Source: h.Revision.Source, Entries: h.Entries}, true
}
+6
View File
@@ -9,6 +9,12 @@ require (
)
require (
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
github.com/klauspost/compress v1.20.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
@@ -0,0 +1,170 @@
package pipeline
import (
"context"
"fmt"
"sort"
"sync"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
)
// Status is a step's status for one artifact of a module, read from the module's registration and never from a
// manifest (design 47 G §21, §23): the artifact's own status where its policy names one, else the module's. A step
// neither names is "" (design question Q-RN8, open).
func Status(reg catalog.Registration, artifact, step string) string {
if a, ok := reg.Policy.Artifacts[artifact]; ok {
if s := statusOf(a, step); s != "" {
return s
}
}
return statusOf(reg.Policy.Steps, step)
}
func statusOf(s catalog.StepStatuses, step string) string {
switch step {
case seat.StepResultStepLint:
return s.Lint
case seat.StepResultStepUnit:
return s.Unit
case seat.StepResultStepIntegration:
return s.Integration
case seat.StepResultStepComposition:
return s.Composition
case seat.StepResultStepReview:
return s.Review
case seat.StepResultStepApproval:
return s.Approval
}
return ""
}
// buildStep is the acting rule's step a build seat names for the languages it builds (design 47 G §323).
const buildStep = "build"
// BuildSeat is the build seat that runs lint and the test steps of an artifact of language, read from the catalog half
// of the pipeline's revision view (design 47 G §323): the seat whose acting rule `build` has `on` naming that language,
// as an entry of the half that claims it carries it in `acts`. Two seats naming one language are an error naming both.
func BuildSeat(entries map[string]catalog.CatalogEntry, language string) (string, error) {
var seats []string
for _, e := range entries {
for _, r := range e.Acts {
if r.Step == buildStep && r.On == language && r.Seat != "" && !contains(seats, r.Seat) {
seats = append(seats, r.Seat)
}
}
}
sort.Strings(seats)
switch len(seats) {
case 1:
return seats[0], nil
case 0:
return "", fmt.Errorf("no build seat's acting rule %s names the language %q in the catalog half", buildStep, language)
}
return "", fmt.Errorf("the build seats %v each name the language %q in their acting rule %s", seats, language, buildStep)
}
func contains(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
// BranchView is the runner's view of one watched branch's catalog half (design 47 G §195, §200, §479): a first view, then
// that branch's catalog-revision events. meshcatalogclient.Catalog is one.
type BranchView interface {
Hear(ctx context.Context, data []byte) error
View() (Half, bool)
}
// Half is the catalog half a branch view holds, as the runner reads it: its revision and its entries by module.
type Half struct {
Revision int64
Source int64
Entries map[string]catalog.CatalogEntry
}
// Views is the runner's views of the catalog half, one per watched branch the registrations name (design 47 G §21,
// §195): a branch first named by a registration gets its view made and started by Open, and every catalog-revision
// heard goes to every branch's view, each taking its own branch's alone.
type Views struct {
// Open makes and starts the view of a branch; its error is said in the journal and the branch asked again at the
// next registration heard.
Open func(branch string) (BranchView, error)
Logf func(string, ...any)
mu sync.Mutex
branches map[string]BranchView
}
// Registered takes the watched branches of a module's registration (design 47 G §21): each branch the views do not hold
// yet is opened.
func (v *Views) Registered(reg catalog.Registration) {
for _, b := range reg.Branches {
v.ensure(b)
}
}
func (v *Views) ensure(branch string) {
v.mu.Lock()
defer v.mu.Unlock()
if v.branches == nil {
v.branches = map[string]BranchView{}
}
if _, ok := v.branches[branch]; ok || branch == "" {
return
}
bv, err := v.Open(branch)
if err != nil {
if v.Logf != nil {
v.Logf("the catalog half of the watched branch %s is not followed (asked again at the next registration heard): %v", branch, err)
}
return
}
v.branches[branch] = bv
}
// Branches are the watched branches the views hold, sorted.
func (v *Views) Branches() []string {
v.mu.Lock()
defer v.mu.Unlock()
out := make([]string, 0, len(v.branches))
for b := range v.branches {
out = append(out, b)
}
sort.Strings(out)
return out
}
// View is the catalog half the view of branch holds; false while it has none, when the runner acts on nothing new of
// that branch (design 47 G §200).
func (v *Views) View(branch string) (Half, bool) {
v.mu.Lock()
bv := v.branches[branch]
v.mu.Unlock()
if bv == nil {
return Half{}, false
}
return bv.View()
}
// HearRevision hands one catalog-revision event to every branch's view.
func (v *Views) HearRevision(ctx context.Context, data []byte) error {
v.mu.Lock()
all := make([]BranchView, 0, len(v.branches))
for _, bv := range v.branches {
all = append(all, bv)
}
v.mu.Unlock()
var first error
for _, bv := range all {
if err := bv.Hear(ctx, data); err != nil && first == nil {
first = err
}
}
return first
}
@@ -0,0 +1,85 @@
package pipeline
import (
"context"
"errors"
"strings"
"testing"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// The policy, the build seat by language and the views per watched branch (design 47 G §21, §23, §323, §195).
func TestPolicyAndBuildSeat(t *testing.T) {
reg := catalog.Registration{Module: "m", Branches: []string{"main"}, Policy: catalog.Policy{
Steps: catalog.StepStatuses{Lint: "required", Unit: "required", Integration: "optional", Approval: "required"},
Artifacts: map[string]catalog.StepStatuses{"tools": {Unit: "bypass"}},
}}
for _, c := range []struct{ artifact, step, want string }{
{"program", "unit", "required"},
{"tools", "unit", "bypass"},
{"tools", "integration", "optional"},
{"tools", "approval", "required"},
{"program", "composition", ""},
} {
if got := Status(reg, c.artifact, c.step); got != c.want {
t.Errorf("Status(%s, %s) = %q, want %q", c.artifact, c.step, got, c.want)
}
}
entries := map[string]catalog.CatalogEntry{
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
"mesh-build-node": {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "typescript"}, {Seat: "node-build-node", Step: "check", On: "go"}}},
"app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}}},
}
for _, c := range []struct{ language, want, err string }{
{"go", "node-build-go", ""},
{"typescript", "node-build-node", ""},
{"rust", "", `"rust"`},
} {
got, err := BuildSeat(entries, c.language)
if got != c.want || (c.err == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.err)) {
t.Errorf("BuildSeat(%s) = %q, %v; want %q, an error naming %s", c.language, got, err, c.want, c.err)
}
}
entries["other"] = catalog.CatalogEntry{Acts: []catalog.ActingRule{{Seat: "node-build-other", Step: "build", On: "go"}}}
if _, err := BuildSeat(entries, "go"); err == nil || !strings.Contains(err.Error(), "node-build-other") {
t.Errorf("two seats naming go: %v, want an error naming both", err)
}
heard := map[string]int{}
opened := 0
v := &Views{Open: func(b string) (BranchView, error) {
opened++
if b == "broken" {
return nil, errors.New("refused")
}
return fakeView{branch: b, heard: heard}, nil
}}
v.Registered(reg)
v.Registered(catalog.Registration{Module: "n", Branches: []string{"main", "feature/x", "broken"}})
if got := v.Branches(); strings.Join(got, ",") != "feature/x,main" || opened != 3 {
t.Fatalf("branches %v opened %d, want feature/x and main, three opens", got, opened)
}
v.Registered(catalog.Registration{Module: "n", Branches: []string{"broken"}})
if opened != 4 {
t.Errorf("a branch whose view was refused is opened again at the next registration: %d opens", opened)
}
if err := v.HearRevision(context.Background(), []byte(`{}`)); err != nil || heard["main"] != 1 || heard["feature/x"] != 1 {
t.Errorf("a revision heard reaches every branch's view: %v %v", err, heard)
}
if h, ok := v.View("main"); !ok || h.Revision != 7 {
t.Errorf("View(main) = %v %v", h, ok)
}
if _, ok := v.View("gone"); ok {
t.Error("a branch not watched has no view")
}
}
type fakeView struct {
branch string
heard map[string]int
}
func (f fakeView) Hear(context.Context, []byte) error { f.heard[f.branch]++; return nil }
func (f fakeView) View() (Half, bool) { return Half{Revision: 7}, true }
@@ -0,0 +1,136 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// ErrExists is a Create of a pipeline key that is written already: the second start, which writes nothing (§54).
var ErrExists = errors.New("the pipeline's key is written already")
// Created is the runner's state pipelines as a start writes it: a key written only when it is absent (compare-and-set on
// its absence), ErrExists when it is not.
type Created interface {
Create(key string, value []byte) error
}
// Begin is what a start names: the pipeline's key (watched branch, source revision, module; design 47 G §27, §54) and
// what the catalog half of that branch's view says of the module.
type Begin struct {
Branch string
Source int64
Module string
Commit string
Fp string
Group string
Repository string
Revision int64
}
// Key is a pipeline's key in the state pipelines, `<branch>.<source revision>.<module>`, the branch escaped as a key's
// name is (§27).
func Key(branch string, source int64, module string) string {
return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), module)
}
// Starter starts pipelines (design 47 G §54, §103): it writes a pipeline's key by compare-and-set on its absence, so one
// source revision and module has one pipeline whichever event starts it first, and says pipeline-started for the one it
// wrote.
type Starter struct {
Pipelines Created
// Say says one of the seat's events on its subject.
Say func(subject string, body any) error
Now func() time.Time
Logf func(string, ...any)
}
// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second
// start writes nothing, §54).
func (s *Starter) Start(b Begin) (bool, error) {
if b.Branch == "" || b.Module == "" || b.Source <= 0 {
return false, fmt.Errorf("a pipeline names its watched branch, source revision and module: %+v", b)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
}
key := Key(b.Branch, b.Source, b.Module)
p := seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fp: b.Fp,
Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now,
Steps: []seat.StepResult{}}
raw, err := json.Marshal(p)
if err != nil {
return false, err
}
if err := s.Pipelines.Create(key, raw); errors.Is(err, ErrExists) {
s.logf("%s is started already: this start writes nothing", key)
return false, nil
} else if err != nil {
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module,
Commit: b.Commit, Fp: b.Fp, Group: b.Group, Revision: b.Revision}
commitOrGroup := b.Commit
if b.Group != "" {
commitOrGroup = b.Group
}
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
return true, nil
}
func (s *Starter) logf(format string, args ...any) {
if s.Logf != nil {
s.Logf(format, args...)
}
}
// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds,
// a package A imports (§53).
const outdatedKind = 1
// Published starts, for a package-published, a pipeline for each dependent the catalog's `outdated` names behind for that
// package on that branch by kind 1, keyed by the publication's watched branch and source revision and the dependent
// (design 47 G §53, §334). outdated is the kept call of the verb; view the catalog half of the publication's branch. A
// dependent the half does not hold is said and started with what the event names.
func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) {
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
}
a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact})
if err != nil {
return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err)
}
half, _ := view(ev.Branch)
started := 0
var first error
for _, o := range a.Behind {
if o.Kind != outdatedKind || o.Module == "" {
continue
}
b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision}
if e, ok := half.Entries[o.Module]; ok {
b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository
} else {
s.logf("the catalog half of %s holds no entry of %s, behind %s: its pipeline starts without its commit", ev.Branch, o.Module, ev.Artifact)
}
ok, err := s.Start(b)
if err != nil && first == nil {
first = err
}
if ok {
started++
}
}
return started, first
}
@@ -0,0 +1,87 @@
package pipeline
import (
"encoding/json"
"errors"
"strings"
"testing"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
)
type created map[string][]byte
func (c created) Create(key string, value []byte) error {
if _, ok := c[key]; ok {
return ErrExists
}
c[key] = value
return nil
}
// The starts (design 47 G §54, §103, §334): a pipeline's key written by compare-and-set on its absence, so a second start
// of the same source revision and module writes nothing; a publish starts one pipeline per dependent outdated names
// behind by kind 1, keyed by the publishing branch and revision.
func TestStarts(t *testing.T) {
store := created{}
var said []string
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
ev := body.(seat.PipelineStarted)
said = append(said, subject+" "+ev.Pipeline)
return nil
}}
view := func(branch string) (Half, bool) {
return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true
}
var asked catalog.OutdatedInput
outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
asked = in
return catalog.OutdatedAnswer{Behind: []catalog.Outdated{
{Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil
}
n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view)
if err != nil || n != 2 {
t.Fatalf("a publish started %d (%v), want the two dependents behind by kind 1", n, err)
}
if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" {
t.Errorf("outdated asked %+v, want the package on its branch", asked)
}
var p seat.Pipeline
if err := json.Unmarshal(store["main.41.app"], &p); err != nil || p.Commit != "c1" || p.Fp != "src2:aa" || p.Revision != 9 ||
p.Outcome != seat.PipelineOutcomeRunning || p.Source != 41 {
t.Errorf("the pipeline main.41.app = %+v (%v)", p, err)
}
if _, ok := store["main.41.ordered"]; ok {
t.Error("an ordering edge starts no pipeline")
}
// The take-in of the same source revision and module is the same pipeline: the second start writes nothing.
before := string(store["main.41.app"])
ok, err := s.Start(Begin{Branch: "main", Source: 41, Module: "app", Commit: "c2"})
if ok || err != nil || string(store["main.41.app"]) != before || len(said) != 2 {
t.Errorf("a second start wrote %v (%v); said %v", ok, err, said)
}
if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c1 main.41.app") {
t.Errorf("pipeline-started said on %q", said[0])
}
// A branch holding a dot is escaped in the key, so the key reads back; a start naming no source revision is refused.
if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Module: "app"}); err != nil {
t.Fatal(err)
}
if b, r, m, ok := seat.ParsePipelinesByBranchRevisionModuleKey(Key("release.1", 3, "app")); !ok || r != "3" || m != "app" || b == "release" {
t.Errorf("the key of release.1 is %q", Key("release.1", 3, "app"))
}
if _, err := s.Start(Begin{Branch: "main", Module: "app"}); err == nil {
t.Error("a start with no source revision is refused")
}
s.Pipelines = failing{}
if _, err := s.Start(Begin{Branch: "main", Source: 5, Module: "app"}); err == nil {
t.Error("a write the bus refuses is an error")
}
}
type failing struct{}
func (failing) Create(string, []byte) error { return errors.New("bus away") }
@@ -49,8 +49,13 @@ func TestManifest(t *testing.T) {
// §27, §85, §330, §355, §77: the old state sections.
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "findings", "first-views"})},
// §27, §85: the step log on an old stream, its outputs in an old objects section.
{"old streams section", equalSets(streams, []string{seat.StateStepLog})},
{"step-log 128 MiB", len(m.Streams) == 1 && m.Streams[0].MaxMiB == 128},
// RN24 (§383, §387): beside it, the finding stream its findings are said on.
{"old streams section", equalSets(streams, []string{seat.StateStepLog, "mesh-delivery-pipeline-findings"})},
{"step-log 128 MiB", len(m.Streams) == 2 && m.Streams[0].MaxMiB == 128},
{"finding stream 7 days 16 MiB", len(m.Streams) == 2 && m.Streams[1].MaxAgeDays == 7 && m.Streams[1].MaxMiB == 16 &&
equalSets(m.Streams[1].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"})},
// RN24 (§368): the version as MAJOR.MINOR, 0.1 with none published.
{"version 0.1", m.Version == "0.1"},
{"old objects section", equalSets(objects, []string{seat.StateStepOutputs})},
{"step-outputs 1 GiB", len(m.Objects) == 1 && m.Objects[0].MaxMiB == 1024},
// §333, §54, §44, §53, §376, §24, §37, §4, §77: what the runner hears.
@@ -58,7 +63,13 @@ func TestManifest(t *testing.T) {
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed",
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
"node-build-image.step-ended", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision"})},
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked"})},
// RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and
// asked again by hand (a hand act).
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act",
// RN3 (§53, §334): a publication's dependents behind.
"seat:mesh-catalog.outdated"})},
{"first-view tools", equalSets(m.Tools, []string{"pipeline_first_views", "pipeline_first_view_again"})},
// RN1/AC5 (§95, §130): the new assigner is delivered before the runner.
{"program needs the assigner's program", slices.Equal(program.Needs, []string{"mesh-delivery-assigner:program"})},
}
+23 -2
View File
@@ -1,6 +1,6 @@
{
"module": "mesh-delivery-pipeline",
"version": "1",
"version": "0.1",
"slug": "pipeline",
"claims": [
{
@@ -22,6 +22,10 @@
]
}
],
"tools": [
"pipeline_first_views",
"pipeline_first_view_again"
],
"consumes": [
"mesh-delivery-indexer.take-in.taken",
"mesh-builds.package-published",
@@ -32,7 +36,14 @@
"node-build-image.step-ended",
"mesh-wants.step-ended",
"mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision"
"mesh-catalog.catalog-revision",
"mesh-conditions.findings-asked"
],
"invokes": [
"seat:mesh-catalog.registrations",
"seat:mesh-catalog.revision",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-catalog.outdated"
],
"state": [
{
@@ -63,6 +74,16 @@
"step-log.**"
],
"max-mib": 128
},
{
"name": "mesh-delivery-pipeline-findings",
"events": [
"finding-written.*",
"finding-cleared.*",
"findings-stated.*"
],
"max-age-days": 7,
"max-mib": 16
}
],
"objects": [