diff --git a/contracts/go/seats/meshpipelinerunner/contract.go b/contracts/go/seats/meshpipelinerunner/contract.go index cdb26625..7b0bfe66 100644 --- a/contracts/go/seats/meshpipelinerunner/contract.go +++ b/contracts/go/seats/meshpipelinerunner/contract.go @@ -391,6 +391,7 @@ const ( // The values of LogEntry's step. const ( + LogEntryStepBuild = "build" LogEntryStepLint = "lint" LogEntryStepUnit = "unit" LogEntryStepIntegration = "integration" @@ -521,6 +522,7 @@ const ( // The values of PipelineStartedStepsItem's step. const ( + PipelineStartedStepsItemStepBuild = "build" PipelineStartedStepsItemStepLint = "lint" PipelineStartedStepsItemStepUnit = "unit" PipelineStartedStepsItemStepIntegration = "integration" @@ -651,6 +653,7 @@ const ( // The values of StepResult's step. const ( + StepResultStepBuild = "build" StepResultStepLint = "lint" StepResultStepUnit = "unit" StepResultStepIntegration = "integration" @@ -715,6 +718,7 @@ const ( // The values of Wait's step. const ( + WaitStepBuild = "build" WaitStepLint = "lint" WaitStepUnit = "unit" WaitStepIntegration = "integration" @@ -817,6 +821,7 @@ type WhyCarriedInput struct { // The values of WhyCarriedInput's step. const ( + WhyCarriedInputStepBuild = "build" WhyCarriedInputStepLint = "lint" WhyCarriedInputStepUnit = "unit" WhyCarriedInputStepIntegration = "integration" @@ -878,6 +883,7 @@ type RerunInput struct { // The values of RerunInput's step. const ( + RerunInputStepBuild = "build" RerunInputStepLint = "lint" RerunInputStepUnit = "unit" RerunInputStepIntegration = "integration" @@ -904,6 +910,7 @@ type AcceptInput struct { // The values of AcceptInput's step. const ( + AcceptInputStepBuild = "build" AcceptInputStepLint = "lint" AcceptInputStepUnit = "unit" AcceptInputStepIntegration = "integration" diff --git a/contracts/seats/mesh-pipeline-runner/seat.json b/contracts/seats/mesh-pipeline-runner/seat.json index d6b36a15..01778d80 100644 --- a/contracts/seats/mesh-pipeline-runner/seat.json +++ b/contracts/seats/mesh-pipeline-runner/seat.json @@ -16,10 +16,6 @@ { "name": "run", "description": "The run on request of one or more members, each a module at a named commit, through their pipelines; delivers nothing, replacing the checker's try (ADR 0666 section 18). Says run-started.", - "callers": [ - "sessions", - "operator" - ], "input": { "type": "object", "properties": { @@ -61,10 +57,6 @@ { "name": "start", "description": "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log.", - "callers": [ - "operator", - "healers" - ], "input": { "type": "object", "properties": { @@ -101,11 +93,6 @@ { "name": "show", "description": "One pipeline whole: its steps, its step log and its outputs, with the stores' fill. Reads only.", - "callers": [ - "persons", - "sessions", - "healers" - ], "input": { "type": "object", "properties": { @@ -153,11 +140,6 @@ { "name": "list", "description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill. Reads only.", - "callers": [ - "persons", - "sessions", - "healers" - ], "input": { "type": "object", "properties": { @@ -200,11 +182,6 @@ { "name": "stands", "description": "What stands still, and since when: each step not ended and each wait held. Reads only.", - "callers": [ - "persons", - "sessions", - "healers" - ], "input": { "type": "object", "properties": {} @@ -228,11 +205,6 @@ { "name": "why-carried", "description": "Where a step's result was carried from, with the kept results it rests on. Reads only.", - "callers": [ - "persons", - "sessions", - "healers" - ], "input": { "type": "object", "properties": { @@ -243,6 +215,7 @@ "step": { "type": "string", "enum": [ + "build", "lint", "unit", "integration", @@ -279,11 +252,6 @@ { "name": "kept", "description": "The kept results, by digest, package or level, with the store's fill. Reads only.", - "callers": [ - "persons", - "sessions", - "healers" - ], "input": { "type": "object", "properties": { @@ -320,9 +288,6 @@ { "name": "review", "description": "A code review's result for a commit and fingerprint, taken from the reviewer's account and refused from the commit's author (ADR 0666 section 22).", - "callers": [ - "sessions" - ], "input": { "type": "object", "properties": { @@ -373,10 +338,6 @@ { "name": "rerun", "description": "One step again; its step-ended says it is a rerun and the caller the runtime proved. Written in the hand-act log.", - "callers": [ - "operator", - "healers" - ], "input": { "type": "object", "properties": { @@ -387,6 +348,7 @@ "step": { "type": "string", "enum": [ + "build", "lint", "unit", "integration", @@ -428,9 +390,6 @@ { "name": "accept", "description": "A failed optional step accepted, with who and why, in the hand-act log.", - "callers": [ - "operator" - ], "input": { "type": "object", "properties": { @@ -441,6 +400,7 @@ "step": { "type": "string", "enum": [ + "build", "lint", "unit", "integration", @@ -479,10 +439,6 @@ { "name": "cancel", "description": "A pipeline ended cancelled, its waiting asks dropped by each build seat's drop. Written in the hand-act log.", - "callers": [ - "operator", - "healers" - ], "input": { "type": "object", "properties": { @@ -516,9 +472,6 @@ { "name": "ask-again", "description": "A declined approval step asked again (ADR 0666 section 10). Written in the hand-act log.", - "callers": [ - "operator" - ], "input": { "type": "object", "properties": { @@ -821,6 +774,7 @@ "step": { "type": "string", "enum": [ + "build", "lint", "unit", "integration", @@ -1206,6 +1160,7 @@ "step": { "type": "string", "enum": [ + "build", "lint", "unit", "integration", @@ -1456,6 +1411,7 @@ "step": { "type": "string", "enum": [ + "build", "lint", "unit", "integration", @@ -1516,6 +1472,7 @@ "step": { "type": "string", "enum": [ + "build", "lint", "unit", "integration", diff --git a/contracts/ts/seats/meshpipelinerunner/index.d.ts b/contracts/ts/seats/meshpipelinerunner/index.d.ts index f270727c..73c3dcb1 100644 --- a/contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -164,7 +164,7 @@ export interface LogEntry { /** the output's name in step-outputs */ "output"?: string; "pipeline": string; - "step": "lint" | "unit" | "integration" | "composition" | "review" | "approval"; + "step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval"; "what": "started" | "ended"; } @@ -251,7 +251,7 @@ export interface PipelineEnded { export interface PipelineStartedStepsItem { "artifact"?: string; "status": "required" | "optional" | "bypass"; - "step": "lint" | "unit" | "integration" | "composition" | "review" | "approval"; + "step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval"; } /** a pipeline's start */ @@ -342,7 +342,7 @@ export interface StepResult { "started"?: string; /** the step's status in the module's policy */ "status": "required" | "optional" | "bypass"; - "step": "lint" | "unit" | "integration" | "composition" | "review" | "approval"; + "step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval"; "tests"?: TestCounts; /** what the outcome rests on: the part not heard, the seat with no holder, what a waiting step waits on */ "why"?: string; @@ -389,7 +389,7 @@ export interface Wait { /** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */ "source"?: number; /** a pipeline step a wait inside a pipeline is in; pipeline is the acting step of design 47 G 203, for a wait of the entries the runner holds for its new version (design 47 G 534) */ - "step": "lint" | "unit" | "integration" | "composition" | "review" | "approval" | "pipeline"; + "step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval" | "pipeline"; "v": number; /** for a wait on a person, who: the operator, a reviewer */ "who"?: string; @@ -474,7 +474,7 @@ export interface WhyCarriedInput { "artifact"?: string; /** the pipeline's key */ "pipeline": string; - "step": "lint" | "unit" | "integration" | "composition" | "review" | "approval"; + "step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval"; } /** is what why-carried answers. */ @@ -517,7 +517,7 @@ export interface RerunInput { "artifact"?: string; /** the pipeline's key */ "pipeline": string; - "step": "lint" | "unit" | "integration" | "composition" | "review" | "approval"; + "step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval"; /** why, written with the caller the runtime proves in the hand-act log */ "why": string; } @@ -533,7 +533,7 @@ export interface AcceptInput { "artifact"?: string; /** the pipeline's key */ "pipeline": string; - "step": "lint" | "unit" | "integration" | "composition" | "review" | "approval"; + "step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval"; /** why, written with the caller the runtime proves in the hand-act log */ "why": string; } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/approval.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/approval.go new file mode 100644 index 00000000..ff6cae89 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/approval.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// toolCaller is a kept call of a seat's verb on the bus (process.Conn.Call). +type toolCaller func(key string, body any) (json.RawMessage, error) + +// routerBucket is the bucket of the router's asks: the state `asks` of the seat operator-channel (the contract's StateAsks) in +// the bucket of the seat's holder on the seat list, which the seat mesh-module-assigner's verb assignments answers with seats +// (design 47 G §700, A-74); no module's name is in the program. (The indexer's function, to move into mesh-sdk.) +func routerBucket(call toolCaller) (string, error) { + raw, err := call("seat:"+mma.Seat+"."+mma.VerbAssignments, mma.AssignmentsInput{Seats: mma.AssignmentsInputSeatsTrue}) + if err != nil { + return "", fmt.Errorf("who holds the seat %s is not asked of the seat %s: %w", oc.Seat, mma.Seat, err) + } + var held struct { + Seats []mma.SeatHolders `json:"seats"` + } + if err := json.Unmarshal(raw, &held); err != nil { + return "", fmt.Errorf("the answer of %s.%s is not its seat's: %w", mma.Seat, mma.VerbAssignments, err) + } + for _, s := range held.Seats { + if s.Seat == oc.Seat && len(s.Holders) > 0 { + return s.Holders[0].Module + "_" + oc.StateAsks, nil + } + } + return "", fmt.Errorf("no module holds the seat %s on the seat list, so the router's record of an ask is not read", oc.Seat) +} + +// routerRecordOf is Hearer.Where: the router's own record of an ask, read under the runner's module's name through +// operatorchannelclient, whose Record is a get of that key (`.` of the seat's state asks) on the process's own bus +// connection (design 47 G §700, A-89): never the decided event's body. +func routerRecordOf(module string, call toolCaller, record func(bucket, id string) (*stdio.StateEntry, error)) func(id string) (string, *oc.Warrant, error) { + client := &operatorchannelclient.Client{Module: module, Record: func(id string) (*stdio.StateEntry, error) { + bucket, err := routerBucket(call) + if err != nil { + return nil, err + } + return record(bucket, id) + }} + return client.Where +} + +// busAsker publishes the runner's approval ask on the seat operator-channel's accept subject under the runner's own name, its +// duplicate id the ask's own, so an ask published twice is kept once (§37). +type busAsker struct { + conn *nats.Conn + cred process.Account +} + +func (b busAsker) Ask(a oc.Ask) error { + data, err := json.Marshal(a) + if err != nil { + return err + } + msg := nats.NewMsg(oc.AcceptSubjectAsk(b.cred.Module)) + msg.Header.Set("x-event-id", a.ID) + msg.Header.Set("x-source", b.cred.Module) + msg.Header.Set("x-node", b.cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Header.Set(nats.MsgIdHdr, a.ID) + msg.Data = data + return b.conn.PublishMsg(msg) +} + +var _ pipeline.Asker = busAsker{} + +// hearDecided wakes the runner on the operator-channel's decided for its own asks (§37, A-89): the warrant is read from the +// router's record, so an event the record does not back yet is heard again. +func hearDecided(hub *process.EventHub, h *pipeline.Hearer, module string, logf func(string, ...any)) error { + return hub.Handle(oc.SubjectDecided(module), func(m jetstream.Msg) { + var ev oc.Warrant + if err := json.Unmarshal(m.Data(), &ev); err != nil { + logf("%s holds what is no decided event, dropped: %v", m.Subject(), err) + _ = m.Term() + return + } + _, err := h.OnDecided(ev) + switch { + case errors.Is(err, pipeline.ErrRefused), errors.Is(err, pipeline.ErrNoAsk): + logf("the decided of ask %s takes nothing: %v", ev.Ask, err) + _ = m.Term() + case err != nil: + logf("the decided of ask %s is not taken yet, heard again: %v", ev.Ask, err) + _ = m.Nak() + default: + _ = m.Ack() + } + }) +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/asks.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/asks.go new file mode 100644 index 00000000..e58e5e06 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/asks.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "encoding/json" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// busStepAsker puts a step's ask on a build seat's work queue at the seat's accept subject, under the runner's own name; its +// duplicate id is the ask's own, so an ask put twice is kept once (design 47 G §22, §24). +type busStepAsker struct { + ctx context.Context + js jetstream.JetStream + cred process.Account +} + +func (b busStepAsker) AskStep(accept, id string, body any) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + msg := nats.NewMsg(accept) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", b.cred.Module) + msg.Header.Set("x-node", b.cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data = data + _, err = b.js.PublishMsg(b.ctx, msg) + return err +} + +var _ pipeline.StepAsker = busStepAsker{} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index c9b95273..23a357aa 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -11,6 +11,7 @@ import ( "log" "os" "os/signal" + "sync/atomic" "syscall" "github.com/nats-io/nats.go" @@ -48,6 +49,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + kept, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StateKept), "keep a package's passing result") + if err != nil { + return err + } + // The bound of each ask a build seat's holder must take: a key with its own time to live, its expiry heard (§535, A-77). + bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound an ask no holder takes") + if err != nil { + return err + } + // The toolchain of each language as heard agreed, for the steps' asks (§416, A-104). + toolchains, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "toolchains"), "name a step's toolchain") + if err != nil { + return err + } // The registrations and each watched branch's catalog half, first views then events (§21, §195, §200): until a // branch's view is held the runner acts on nothing new of it, and says so. calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) @@ -56,26 +71,77 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer calls.Close() hub := process.NewEventHub(logger.Printf) - rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, logger.Printf) + // A branch's view holding its first half asks again the steps placed in its running pipelines that were never asked + // (§88); the starter is made after the views, so it is reached through a pointer set once it is. + var resumer atomic.Pointer[func(string)] + rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) { + if f := resumer.Load(); f != nil { + (*f)(branch) + } + }, logger.Printf) if err != nil { return err } // A pipeline starts by compare-and-set on its key's absence and says pipeline-started (§54, §103); a publish starts // one per dependent behind (§53, §334). - starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, - Say: func(subject string, body any) error { - data, err := json.Marshal(body) - if err == nil { - _, err = js.Publish(ctx, subject, data) - } - return err - }} + say := func(subject string, body any) error { + data, err := json.Marshal(body) + if err == nil { + _, err = js.Publish(ctx, subject, data) + } + return err + } + hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf) + hearer.Asker = busAsker{conn: conn, cred: cred} + hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask) + // Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked + // for its step. + starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say, + Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module, + Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains)} + resume := func(branch string) { + half, ok := rv.Catalog.View(branch) + if !ok { + return + } + n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half) + if err != nil { + logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err) + return + } + logger.Printf("the steps placed on %s and not yet asked: %d asked again", branch, n) + } + resumer.Store(&resume) + if err := hearStepEnded(hub, hearer, logger.Printf); err != nil { + return err + } + if err := hearToolchain(ctx, hub, toolchains, logger.Printf); err != nil { + return err + } + if err := hearAskTaken(ctx, hub, bounds, logger.Printf); err != nil { + return err + } + if err := watchBounds(ctx, bounds, hearer, logger.Printf); err != nil { + return err + } + if err := hearDecided(hub, hearer, cred.Module, logger.Printf); err != nil { + return err + } if err := hearPublished(ctx, hub, starter, calls, rv.Catalog, logger.Printf); err != nil { return err } + if err := hearTakenIn(hub, starter, rv.Catalog, logger.Printf); err != nil { + return err + } if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil { logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err) } + // The seat's write verbs are answered here, by the process that holds the pipelines (A-107). + stopVerbs, err := answerVerbs(calls, writeVerbs(ctx, hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls, bounds)) + if err != nil { + return err + } + defer stopVerbs() go func() { <-ctx.Done(); hub.Stop() }() logger.Printf("holding %s", seat.Seat) <-ctx.Done() diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/router_record_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/router_record_test.go new file mode 100644 index 00000000..fea5a54d --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/router_record_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// Design 47 G §700, A-74: the router's record of an ask is read under the indexer's module's name from the state asks of the +// seat operator-channel, in the bucket of the seat's holder on the seat list, and carries the ask's warrant. +func TestTheRoutersRecordIsReadFromTheBucketOfTheSeatsHolder(t *testing.T) { + var asked []string + call := func(key string, body any) (json.RawMessage, error) { + asked = append(asked, key) + return json.RawMessage(`{"seats":[{"seat":"node-packet-filter","scope":"node","node":"n1","holders":[{"module":"pf","node":"n1"}]},` + + `{"seat":"operator-channel","scope":"mesh","holders":[{"module":"ops-router","node":"n2"}]}],"holds":2}`), nil + } + var bucket, id string + record := func(b, i string) (*stdio.StateEntry, error) { + bucket, id = b, i + return &stdio.StateEntry{Key: "mesh-delivery-pipeline." + i, Value: json.RawMessage(`{"state":"chosen","warrant":{"ask":"a1","asker":"mesh-delivery-pipeline","choice":"approve"}}`)}, nil + } + state, w, err := routerRecordOf("mesh-delivery-pipeline", call, record)("a1") + if err != nil || state != "chosen" || w == nil || w.Ask != "a1" { + t.Fatalf("Where = %q, %+v, %v; want the state chosen and the warrant of a1", state, w, err) + } + if bucket != "ops-router_asks" || id != "a1" || len(asked) != 1 || asked[0] != "seat:mesh-module-assigner.assignments" { + t.Errorf("read %q key %q after asking %v; want the holder's bucket ops-router_asks, the id, one question of the seat mesh-module-assigner", bucket, id, asked) + } + // No holder on the seat list: nothing is read, and the warrant stays unapplied with the reason said. + none := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"seats":[],"holds":0}`), nil } + if _, _, err := routerRecordOf("mesh-delivery-pipeline", none, record)("a1"); err == nil || !strings.Contains(err.Error(), "no module holds the seat operator-channel") { + t.Errorf("with no holder Where answered %v; want the reason", err) + } +} + +// The asker the router and the decided event name is the runner's own module (its bus account's), never a constant: a hearer +// that named another refused every decided of its own asks. +func TestTheHearerIsNamedByTheRunnersOwnModule(t *testing.T) { + h := newHearer(context.Background(), nil, process.Account{Module: "the-runner"}, nil, nil, nil, nil) + if h.Name != "the-runner" { + t.Errorf("the hearer's name is %q, want its account's module", h.Name) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go new file mode 100644 index 00000000..a143be7d --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -0,0 +1,190 @@ +//go:build integration + +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type syncBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.Write(p) +} +func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() } + +// The runner whole against the real bus (design 47 G §22, §44, §54, §70, §482, A-107): started as its process is, it takes a +// take-in.taken and places a pipeline of the registered module, asks its build of the seat mesh-builds, takes the step-ended +// of the build, and ends the placed review step when a session calls `review`, answered by the process with the caller the +// bus proved; the pipeline then ends passed. +func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + const module = "mesh-delivery-pipeline" + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { + t.Fatal(err) + } + } + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}) + if err != nil { + t.Fatal(err) + } + if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + asks, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}) + if err != nil { + t.Fatal(err) + } + + // The seats the runner calls, answered as their holders do. + reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional"}}} + answer := func(subject string, body any) { + if _, err := nc.Subscribe(subject, func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": body}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + } + answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: []catalog.Registration{reg}}) + answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{NoneYet: true}) + answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) + + dir := t.TempDir() + broker := filepath.Join(dir, "broker") + raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"}) + if err := os.WriteFile(broker, raw, 0o600); err != nil { + t.Fatal(err) + } + logs := &syncBuffer{} + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { + done <- run(runCtx, func(k string) string { + if k == process.BrokerFileEnv { + return broker + } + return "" + }, logs) + }() + defer func() { stop(); <-done }() + + pipelines, err := js.KeyValue(ctx, module+"_pipelines") + if err != nil { + t.Fatal(err) + } + // A take-in.taken names one entry of the source revision; the runner places the pipeline of the registered module. + taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m.1", Kind: "taken", Branch: "main", Source: 3, + Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}) + const key = "main.3.app" + var p seat.Pipeline + waitFor(t, ctx, logs, func() bool { + // Said until the runner holds the registration, so the pipeline is placed with its policy's steps. + if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil { + t.Fatal(err) + } + return readPipeline(ctx, pipelines, key, &p) && len(p.Steps) > 1 + }) + build := stepOf(p, seat.StepResultStepBuild) + if build.Outcome != seat.StepResultOutcomeRunning || build.Ask == "" { + t.Fatalf("the build step = %+v", build) + } + // The build is asked of the seat mesh-builds: its ask is on its work queue. + waitFor(t, ctx, logs, func() bool { info, err := asks.Info(ctx); return err == nil && info.State.Msgs >= 1 }) + + // A session's review, while the build still runs, is answered by the runner's process, the caller the one the subject names. + in, _ := json.Marshal(seat.ReviewInput{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Outcome: "passed", Said: "reads well"}) + var reply struct { + Result seat.ReviewAnswer `json:"result"` + Error string `json:"error"` + } + waitFor(t, ctx, logs, func() bool { + m, err := nc.Request("mesh.seat.mesh-pipeline-runner.call.review.agent~dev", in, 2*time.Second) + return err == nil && json.Unmarshal(m.Data, &reply) == nil + }) + if reply.Error != "" || reply.Result.Pipeline != key || reply.Result.Result.By != "agent.dev" || reply.Result.Result.Outcome != "passed" { + t.Fatalf("review answered %+v (%s)", reply.Result, reply.Error) + } + if !readPipeline(ctx, pipelines, key, &p) || stepOf(p, seat.StepResultStepReview).By != "agent.dev" || p.Outcome != seat.PipelineOutcomeRunning { + t.Fatalf("pipeline after the review = %+v\n%s", p, logs.String()) + } + + // The seat's holder says the build ended; the runner takes it from the node's subject. + ended, _ := json.Marshal(builds.StepEnded{ID: build.Ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed", + Entries: []builds.BuildEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}) + if _, err := js.Publish(ctx, "mesh.seat.mesh-builds.event.step-ended.n1", ended); err != nil { + t.Fatal(err) + } + waitFor(t, ctx, logs, func() bool { + return readPipeline(ctx, pipelines, key, &p) && stepOf(p, seat.StepResultStepBuild).Outcome == seat.StepResultOutcomePassed + }) + + waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed }) +} + +func stepOf(p seat.Pipeline, step string) seat.StepResult { + for _, s := range p.Steps { + if s.Step == step { + return s + } + } + return seat.StepResult{} +} + +func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *seat.Pipeline) bool { + e, err := kv.Get(ctx, key) + if err != nil { + return false + } + *p = seat.Pipeline{} + return json.Unmarshal(e.Value(), p) == nil +} + +// waitFor asks cond until it holds or the test's context ends, the module's journal in the failure. +func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bool) { + t.Helper() + for ctx.Err() == nil { + if cond() { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("it did not come about in time; the runner said:\n%s", logs.String()) +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index a2341544..6942aba5 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -69,3 +69,24 @@ func hearPublished(ctx context.Context, hub *process.EventHub, starter *pipeline }() }) } + +// hearTakenIn starts the pipelines of each take-in.taken heard (§44, §54): one per entry, the message acknowledged once +// they are written, and heard again when a write failed, so the keys already written start nothing twice. +func hearTakenIn(hub *process.EventHub, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { + return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { + var ev catalog.TakeInTaken + if err := json.Unmarshal(m.Data(), &ev); err != nil { + logf("%s is no take-in.taken: %v", m.Subject(), err) + _ = m.Ack() + return + } + n, err := starter.TakenIn(ev, v.View) + if err != nil { + logf("the take-in %s of %s: %d pipelines started, then %v; heard again", ev.Key, ev.Branch, n, err) + _ = m.Nak() + return + } + logf("the take-in %s of %s started %d pipelines", ev.Key, ev.Branch, n) + _ = m.Ack() + }) +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go new file mode 100644 index 00000000..8e6c7ae4 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -0,0 +1,270 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/nats-io/nats.go/jetstream" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// pipelineStore is the state pipelines as the hearer reads and writes it: a write by compare-and-set on the revision read (§27). +type pipelineStore struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (s pipelineStore) Get(key string) (seat.Pipeline, uint64, error) { + e, err := s.kv.Get(s.ctx, key) + if err != nil { + return seat.Pipeline{}, 0, err + } + var p seat.Pipeline + if err := json.Unmarshal(e.Value(), &p); err != nil { + return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) + } + return p, e.Revision(), nil +} + +func (s pipelineStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) { + raw, err := json.Marshal(p) + if err != nil { + return 0, err + } + rev, err := s.kv.Update(s.ctx, key, raw, revision) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { + return 0, pipeline.ErrChanged + } + return rev, err +} + +func (s pipelineStore) Keys(filter string) ([]string, error) { + all, err := s.kv.Keys(s.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + var out []string + want := strings.Split(filter, ".") + for _, k := range all { + got := strings.Split(k, ".") + if len(got) != len(want) { + continue + } + ok := true + for i := range want { + ok = ok && (want[i] == "*" || want[i] == got[i]) + } + if ok { + out = append(out, k) + } + } + return out, err +} + +// newHearer is the hearer over the runner's own states: pipelines, kept, the step log on its stream and step-outputs on its +// objects (§27). Each store is opened where it is used, so one the grants part has not made yet fails that write loudly. +func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue, + say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer { + return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module, + Log: func(e seat.LogEntry) error { + return say(cred.EventSubject("step-log."+keys.EscapeName(e.Pipeline)), e) + }, + Keep: func(key string, k seat.KeptResult) error { + raw, err := json.Marshal(k) + if err == nil { + _, err = kept.Put(ctx, key, raw) + } + return err + }, + Output: func(key string, data []byte) error { + o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs)) + if err != nil { + return fmt.Errorf("the objects %s are not open: %w", seat.StateStepOutputs, err) + } + _, err = o.PutBytes(ctx, key, data) + return err + }} +} + +// hearStepEnded takes each build seat's step-ended into its pipeline (§24; ADR 0666 §21): refused and said, never taken, +// when it is not as the runner asked it, and heard again when a write failed. +func hearStepEnded(hub *process.EventHub, h *pipeline.Hearer, logf func(string, ...any)) error { + for _, bs := range pipeline.StepSeats { + bs := bs + err := hub.Handle(bs.Prefix+"*", func(m jetstream.Msg) { + decode := bs.Decode + if decode == nil { + decode = func(raw []byte) (pipeline.StepEnded, error) { + var ev pipeline.StepEnded + return ev, json.Unmarshal(raw, &ev) + } + } + ev, err := decode(m.Data()) + if err != nil { + logf("%s is no step-ended: %v", m.Subject(), err) + _ = m.Term() + return + } + _, err = h.OnStepEnded(bs, m.Subject(), ev) + switch { + case errors.Is(err, pipeline.ErrRefused): + logf("%v", err) + _ = m.Term() + case err != nil: + logf("the step-ended of ask %s is not taken yet, heard again: %v", ev.ID, err) + _ = m.Nak() + default: + _ = m.Ack() + } + }) + if err != nil { + return err + } + } + return nil +} + +// hearAskTaken deletes the bound of each ask a build seat's holder took (§602, §647): the ask is answered, so its five minutes +// end with no finding. +func hearAskTaken(ctx context.Context, hub *process.EventHub, bounds jetstream.KeyValue, logf func(string, ...any)) error { + for _, ts := range pipeline.TakenSeats { + ts := ts + err := hub.Handle(ts.Subject, func(m jetstream.Msg) { + var ev nodebuildgo.AskTaken + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { + logf("%s is no ask-taken: %v", m.Subject(), err) + _ = m.Term() + return + } + if err := bounds.Delete(ctx, ev.Ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + logf("the bound of ask %s, taken on %s, is not deleted, heard again: %v", ev.Ask, ev.Node, err) + _ = m.Nak() + return + } + _ = m.Ack() + }) + if err != nil { + return err + } + } + return nil +} + +// armBound is the bound of an ask: a key of the state bounds with its own time to live, named for the ask. +func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm { + return func(ask string) error { + _, err := bounds.Create(ctx, ask, []byte(`{"ask":"`+ask+`"}`), jetstream.KeyTTL(pipeline.AskBound)) + if errors.Is(err, jetstream.ErrKeyExists) { + return nil + } + return err + } +} + +// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32, +// §535, A-77): no timer in the process. +func watchBounds(ctx context.Context, bounds jetstream.KeyValue, h *pipeline.Hearer, logf func(string, ...any)) error { + w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + return err + } + go func() { + for { + select { + case <-ctx.Done(): + _ = w.Stop() + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsAskKey(e.Key()) { + continue + } + if err := h.NotTaken(e.Key(), pipeline.AskBound); err != nil { + logf("the step of ask %s, which no holder took, is not ended: %v", e.Key(), err) + } + } + } + }() + return nil +} + +// keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a +// name or lies beneath it, in one pass over the bucket. +func keptReader(ctx context.Context, kept jetstream.KeyValue) func([]string) (map[string][]nodebuildgo.KeptResult, error) { + return func(pkgs []string) (map[string][]nodebuildgo.KeptResult, error) { + w, err := kept.WatchAll(ctx, jetstream.IgnoreDeletes()) + if err != nil { + return nil, err + } + defer w.Stop() + out := map[string][]nodebuildgo.KeptResult{} + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case e, ok := <-w.Updates(): + if !ok { + return out, nil + } + if e == nil { // the end of what the bucket holds now + return out, nil + } + var k nodebuildgo.KeptResult + if err := json.Unmarshal(e.Value(), &k); err != nil { + continue + } + for _, name := range pkgs { + if pipeline.InPackage(k.Package, name) { + out[name] = append(out[name], k) + } + } + } + } + } +} + +// toolchainReader reads the toolchain of a language as the runner heard it agreed, from its state toolchains (A-104); nil +// when none was heard. +func toolchainReader(ctx context.Context, toolchains jetstream.KeyValue) func(string) (*nodebuildgo.ToolchainUsed, error) { + return func(language string) (*nodebuildgo.ToolchainUsed, error) { + e, err := toolchains.Get(ctx, keys.EscapeName(language)) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return pipeline.ToolchainOf(e.Value()) + } +} + +// hearToolchain keeps each toolchain-agreed heard as `` in the state toolchains (A-104): the newest of a language. +func hearToolchain(ctx context.Context, hub *process.EventHub, toolchains jetstream.KeyValue, logf func(string, ...any)) error { + return hub.Handle(builds.SubjectToolchainAgreed, func(m jetstream.Msg) { + t, err := pipeline.ToolchainOf(m.Data()) + if err != nil || t == nil { + logf("%s is no toolchain-agreed: %v", m.Subject(), err) + _ = m.Term() + return + } + if _, err := toolchains.Put(ctx, keys.EscapeName(t.Language), m.Data()); err != nil { + logf("the toolchain of %s is not kept, heard again: %v", t.Language, err) + _ = m.Nak() + return + } + _ = m.Ack() + }) +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go new file mode 100644 index 00000000..b082fef0 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// verbOf makes the handler of one verb from its function of the proved caller and the decoded input: the caller is the bus +// user the call's subject names, and a call naming none is refused (design 47 G §394, A-107). +func verbOf[In any](do func(caller string, in In) (any, error)) process.SeatHandler { + return func(caller string, body json.RawMessage) (any, error) { + if caller == "" { + return nil, errors.New("the call names no caller the bus proved: it is refused") + } + var in In + if len(body) > 0 { + if err := json.Unmarshal(body, &in); err != nil { + return nil, fmt.Errorf("the arguments do not fit the verb's input: %w", err) + } + } + return do(caller, in) + } +} + +// answerVerbs answers the seat's write verbs `review`, `rerun`, `accept` and `cancel` over the bus, each writing the state +// pipelines by compare-and-set (design 47 G §22, §70, §482, A-107). The returned stop ends the answers. +func answerVerbs(conn *process.Conn, v pipeline.Verbs) (func(), error) { + return conn.AnswerSeat(seat.Seat, map[string]process.SeatHandler{ + seat.VerbReview: verbOf(func(c string, in seat.ReviewInput) (any, error) { return v.Review(c, in) }), + seat.VerbRerun: verbOf(func(c string, in seat.RerunInput) (any, error) { return v.Rerun(c, in) }), + seat.VerbAccept: verbOf(func(c string, in seat.AcceptInput) (any, error) { return v.Accept(c, in) }), + seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }), + }) +} + +// dropBound deletes the bound of an ask a cancelled pipeline no longer waits on. +func dropBound(ctx context.Context, bounds jetstream.KeyValue) func(string) error { + return func(ask string) error { + if err := bounds.Delete(ctx, ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil + } +} + +// writeVerbs are the verbs over the hearer and the starter, their hand acts recorded through the seat mesh-hand-acts. +func writeVerbs(ctx context.Context, h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool), + calls *process.Conn, bounds jetstream.KeyValue) pipeline.Verbs { + return pipeline.Verbs{Hearer: h, Starter: s, View: view, Acts: meshhandactsclient.Recorder{Ask: calls.Call}, Drop: dropBound(ctx, bounds)} +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go new file mode 100644 index 00000000..5f85c5f2 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go @@ -0,0 +1,24 @@ +package main + +import ( + "encoding/json" + "testing" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +// A write verb is answered only for a caller the bus proved, and its input is the verb's own (design 47 G §394, A-107). +func TestAVerbIsAnsweredOnlyForAProvedCallerAndItsOwnInput(t *testing.T) { + var got seat.CancelInput + h := verbOf(func(c string, in seat.CancelInput) (any, error) { got = in; return c, nil }) + if _, err := h("", json.RawMessage(`{"pipeline":"p","why":"w"}`)); err == nil { + t.Error("a call naming no proved caller was answered") + } + if _, err := h("operator", json.RawMessage(`{"pipeline":`)); err == nil { + t.Error("a malformed input was answered") + } + out, err := h("operator", json.RawMessage(`{"pipeline":"p","why":"w"}`)) + if err != nil || out != "operator" || got.Pipeline != "p" || got.Why != "w" { + t.Errorf("answer = %v, %v, input %+v", out, err, got) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index 3918bfd5..3228a10c 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -25,7 +25,7 @@ type runnerViews struct { // startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or // after, then start hub once. func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State, - call meshcatalogclient.Call, logf func(string, ...any)) (*runnerViews, error) { + call meshcatalogclient.Call, onReady func(branch string), logf func(string, ...any)) (*runnerViews, error) { findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf) if err != nil { return nil, err @@ -37,7 +37,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr // A watched branch first named by a registration gets its view of the catalog half (CT10), adopted through the reader (design // 47 G §532, A-26): the reader keeps its status, raises and clears its findings through the module's writer, hears the // seat's catalog-revision on the module's consumer and asks its first view at once. - cv := &pipeline.Views{Logf: logf, Open: func(branch string) (pipeline.BranchView, error) { + cv := &pipeline.Views{Logf: logf, OnReady: onReady, Open: func(branch string) (pipeline.BranchView, error) { c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, nil) if err != nil { return nil, err @@ -59,6 +59,8 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr // catalogView is a branch's meshcatalogclient.Catalog as the runner reads it. type catalogView struct{ c *meshcatalogclient.Catalog } +func (v catalogView) Ready() <-chan struct{} { return v.c.Ready() } + func (v catalogView) Hear(ctx context.Context, data []byte) error { return v.c.Hear(ctx, data) } func (v catalogView) View() (pipeline.Half, bool) { diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 1f8869f5..ca25c2c2 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -9,10 +9,16 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 github.com/klauspost/compress v1.20.1 // indirect diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/approval.go b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go new file mode 100644 index 00000000..e0367e8b --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go @@ -0,0 +1,218 @@ +package pipeline + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The approval step is an authorising ask of tier approve, the seat operator-channel's accepted kind ask under the +// runner's own name, its about the module and the watched branch only, so a newer commit's ask replaces the older one's +// (design 47 G §26, §37; ADR 0666 §10). It is asked last, once every other required step passed. +const ( + OptionApprove = "approve" + OptionDecline = "decline" +) + +// About is what an approval ask is about: the module and the watched branch, each a key's name, joined by a dot. +func About(module, branch string) string { + return keys.EscapeName(module) + "." + keys.EscapeName(branch) +} + +// approvalDue is the index of the approval step to ask now (§26): waiting, not bypassed, every other step ended and every +// other required step passed or carried; -1 when none is due. +func approvalDue(p seat.Pipeline) int { + at := -1 + for i, s := range p.Steps { + if s.Step == seat.StepResultStepApproval { + if s.Status != seat.StepResultStatusBypass && s.Outcome == seat.StepResultOutcomeWaiting { + at = i + } + continue + } + if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning { + return -1 + } + if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried { + return -1 + } + } + return at +} + +// ApprovalAsk is the ask of a pipeline's approval step. Its id and its expiry follow from the pipeline and from when the +// step was asked, so the ask built again for the same pipeline is the same ask, digest included: a warrant is held to it. +func ApprovalAsk(p seat.Pipeline, from time.Time) (oc.Ask, error) { + sum := sha256.Sum256([]byte(p.Pipeline + "\n" + p.Commit)) + act := func(do string) (string, error) { + return operatorchannelclient.ActDigest(operatorchannelclient.Act{"pipeline": p.Pipeline, "commit": p.Commit, + "fingerprint": p.Fingerprint, "do": do}) + } + approve, err := act(OptionApprove) + if err != nil { + return oc.Ask{}, err + } + decline, err := act(OptionDecline) + if err != nil { + return oc.Ask{}, err + } + a := oc.Ask{ID: "approval-" + hex.EncodeToString(sum[:12]), About: About(p.Module, p.Branch), Who: operatorchannelclient.Operator, + Headline: trimRunes("Deliver "+p.Module+" from "+p.Branch+"?", operatorchannelclient.HeadlineLength), + Explanation: fmt.Sprintf("Every required step of %s's pipeline at commit %s on %s passed; its policy asks your approval before it is delivered.", p.Module, p.Commit, p.Branch), + Details: "pipeline " + p.Pipeline + ", fingerprint " + p.Fingerprint, + Expires: operatorchannelclient.FormatTime(from.Add(operatorchannelclient.ApproveLasts)), + OnExpiry: "the step ends declined and the commit is not delivered; ask-again asks once more", + Options: []oc.Option{ + {ID: OptionApprove, Label: "Approve", Does: "delivers this commit of " + p.Module, Level: operatorchannelclient.Approve, Binds: approve}, + {ID: OptionDecline, Label: "Decline", Does: "ends the step declined; nothing is delivered", Level: operatorchannelclient.Approve, Binds: decline}, + }} + return a, operatorchannelclient.Check(a, from) +} + +func trimRunes(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n-1]) + "…" +} + +// Asker is the runner's side of the operator-channel: Ask publishes an ask under the runner's name, its duplicate id the +// ask's own, so an ask published twice is kept once. +type Asker interface { + Ask(oc.Ask) error +} + +// askApproval asks p's approval step when it is due, before the pipeline is written: an ask published and then not +// written is published again, the same ask, when its step's end is heard again. +func askApproval(key string, p *seat.Pipeline, from time.Time, asker Asker) (bool, error) { + i := approvalDue(*p) + if i < 0 || asker == nil { + return false, nil + } + a, err := ApprovalAsk(*p, from) + if err != nil { + return false, fmt.Errorf("the approval of %s cannot be asked: %w", key, err) + } + if err := asker.Ask(a); err != nil { + return false, fmt.Errorf("the approval of %s is not asked: %w", key, err) + } + s := &p.Steps[i] + s.Outcome, s.Ask, s.Seat, s.Started = seat.StepResultOutcomeRunning, a.ID, seat.Seat, &from + s.Why = "waiting on the operator" + return true, nil +} + +// ErrNoAsk is a warrant naming no approval step the runner keeps for its about. +var ErrNoAsk = errors.New("no approval step asked under the warrant's ask") + +// decide ends an approval step on its warrant: a chosen approve, held to the ask rebuilt from the step, ends it passed; a +// chosen decline, an expiry, a cancel or a refusal ends it declined; a replacement by a newer commit's ask ends it +// superseded (design 47 G §37). It says whether the step changed. +func decide(p *seat.Pipeline, w oc.Warrant, asker string, now time.Time) (bool, error) { + for i := range p.Steps { + s := &p.Steps[i] + if s.Step != seat.StepResultStepApproval || s.Ask != w.Ask || s.Started == nil { + continue + } + if s.Outcome != seat.StepResultOutcomeRunning { + return false, nil + } + switch w.Outcome { + case oc.WarrantOutcomeChosen: + asked, err := ApprovalAsk(*p, *s.Started) + if err != nil { + return false, err + } + o, err := operatorchannelclient.For(w, asker, asked) + if err != nil { + return false, fmt.Errorf("the warrant for %s is not taken: %w", w.Ask, err) + } + s.Outcome = seat.StepResultOutcomeDeclined + if o.ID == OptionApprove { + s.Outcome = seat.StepResultOutcomePassed + } + if w.By != nil { + s.By = w.By.Identity + } + s.Why = "the operator chose " + o.ID + case oc.WarrantOutcomeReplaced: + s.Outcome, s.Why = seat.StepResultOutcomeSuperseded, "a newer commit's ask replaced it" + default: + s.Outcome, s.Why = seat.StepResultOutcomeDeclined, "the ask ended "+w.Outcome+" "+w.Words + } + s.Ended = &now + return true, nil + } + return false, ErrNoAsk +} + +// ErrNoRecord is a decided event whose ask the router's record holds no warrant for yet: the event is heard again. +var ErrNoRecord = errors.New("the router's record of the ask holds no warrant yet") + +// OnDecided takes the operator-channel's decided for the runner's own asks: it only wakes the runner (design 47 G A-89), which +// reads the warrant from the router's record of the ask through Where, never from the event's body (§37, §700). The approval +// step asked under that ask, among the pipelines of the module and watched branch the ask's about names, ends as decide says. +func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) { + if event.Asker != h.Name { + return seat.Pipeline{}, fmt.Errorf("%w: the decided event is %s's, not %s's", ErrRefused, event.Asker, h.Name) + } + if h.Where == nil { + return seat.Pipeline{}, fmt.Errorf("the runner reads no router's record, so the ask %s is not taken", event.Ask) + } + _, rec, err := h.Where(event.Ask) + if err != nil { + return seat.Pipeline{}, fmt.Errorf("the router's record of the ask %s cannot be read: %w", event.Ask, err) + } + if rec == nil { + return seat.Pipeline{}, fmt.Errorf("%w: %s", ErrNoRecord, event.Ask) + } + w := *rec + module, branch, ok := strings.Cut(w.About, ".") + if !ok { + return seat.Pipeline{}, fmt.Errorf("%w: the warrant for %s is about %q, not a module and a branch", ErrNoAsk, w.Ask, w.About) + } + found, err := h.Store.Keys(branch + ".*." + module) + if err != nil { + return seat.Pipeline{}, err + } + for _, key := range found { + for { + p, rev, err := h.Store.Get(key) + if err != nil { + return seat.Pipeline{}, err + } + now := h.now() + changed, err := decide(&p, w, h.Name, now) + if errors.Is(err, ErrNoAsk) { + break + } + if err != nil || !changed { + return p, err + } + ended := endPipeline(&p, now) + if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return p, err + } + i := stepOfAsk(p, w.Ask) + h.logEnd(key, p.Steps[i], now) + for _, ev := range []string{seat.EventWaitEnded, seat.EventStepEnded} { + if err := h.say(ev, p, &p.Steps[i]); err != nil { + return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", key, ev, err) + } + } + return p, h.said(p, false, ended) + } + } + return seat.Pipeline{}, fmt.Errorf("%w: %s about %s", ErrNoAsk, w.Ask, w.About) +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go new file mode 100644 index 00000000..94d3757f --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go @@ -0,0 +1,95 @@ +package pipeline + +import ( + "errors" + "fmt" + "strings" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + nodebuildimage "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage" + nodebuildnode "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode" +) + +// AskBound is how long an ask on a build seat's work queue may wait for its holder to take it (design 47 G §535, §647): five +// minutes, the bound of a key of the state `bounds` with its own time to live (A-77, A-104), never a timer. +const AskBound = 5 * time.Minute + +// TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it. +type TakenSeat struct { + Seat string + // Subject hears every node's ask-taken of the seat. + Subject string +} + +// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken, +// so no bound is armed for their asks. +var TakenSeats = []TakenSeat{ + {Seat: nodebuildgo.Seat, Subject: nodebuildgo.SubjectAskTakenAll}, + {Seat: nodebuildnode.Seat, Subject: nodebuildnode.SubjectAskTakenAll}, + {Seat: nodebuildimage.Seat, Subject: nodebuildimage.SubjectAskTakenAll}, +} + +// BoundSeat says whether the holder of a seat says ask-taken, so an ask put on its work queue has a bound. +func BoundSeat(name string) bool { + for _, s := range TakenSeats { + if s.Seat == name { + return true + } + } + return false +} + +// Arm puts a bound of one ask's wait for its holder: a key of the state `bounds` named for the ask, written with its own +// time to live. It is armed before the ask is put, so an ask-taken heard at once finds it. +type Arm func(ask string) error + +// NotTaken ends the step asked under ask that no holder took within its bound, the bound's expiry heard (design 47 G §32, +// §535, §647, A-77): the step ends `could not run` naming the seat, so its pipeline ends failed once the others end, never +// a silent wait. A step that ended or was never asked of a seat takes nothing. +func (h *Hearer) NotTaken(ask string, bound time.Duration) error { + pkeys, err := h.Store.Keys("*.*.*") + if err != nil { + return fmt.Errorf("the pipelines are not listed: %w", err) + } + for _, key := range pkeys { + p, _, err := h.Store.Get(key) + if err != nil || p.Outcome != seat.PipelineOutcomeRunning || stepOfAsk(p, ask) < 0 { + continue + } + return h.endNotTaken(key, ask, bound) + } + return nil +} + +func (h *Hearer) endNotTaken(key, ask string, bound time.Duration) error { + for { + p, rev, err := h.Store.Get(key) + if err != nil { + return fmt.Errorf("the pipeline %s is not read: %w", key, err) + } + i := stepOfAsk(p, ask) + if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning { + return nil + } + now := h.now() + s := &p.Steps[i] + s.Outcome, s.Ended = seat.StepResultOutcomeCouldNotRun, &now + s.Why = fmt.Sprintf("the seat %s has no holder that took ask %s within %s", s.Seat, ask, bound) + ended := endPipeline(&p, now) + if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("pipeline %s cannot be written: %w", key, err) + } + h.logEnd(key, *s, now) + if err := h.say(seat.EventStepEnded, p, s); err != nil { + return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err) + } + return h.said(p, false, ended) + } +} + +// IsAskKey says whether a key of the state `bounds` is an ask's bound. +func IsAskKey(key string) bool { return strings.HasPrefix(key, "step-") && !strings.Contains(key, ".") } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go new file mode 100644 index 00000000..6530c35e --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -0,0 +1,168 @@ +package pipeline + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" + "time" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + nodebuildimage "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage" + nodebuildnode "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode" +) + +// StepAsker puts one step's ask on a build seat's work queue, at the subject accept names (the seat's accepted kind `step` +// on its stream `check`, design 47 G §22, §24). An ask asked twice under one id is kept once. +type StepAsker interface { + AskStep(accept, id string, body any) error +} + +// acceptStep is where each build seat is sent its step, from its contract. +var acceptStep = map[string]string{ + builds.Seat: builds.AcceptSubjectStep, + wants.Seat: wants.AcceptSubjectStep, + nodebuildgo.Seat: nodebuildgo.AcceptSubjectCheckStep, + nodebuildnode.Seat: nodebuildnode.AcceptSubjectCheckStep, + nodebuildimage.Seat: nodebuildimage.AcceptSubjectCheckStep, +} + +// buildSteps are the steps an artifact's build seat runs, in the order they are asked (design 47 G §22). +var buildSteps = []string{seat.StepResultStepLint, seat.StepResultStepUnit, seat.StepResultStepIntegration} + +// moduleSteps are the steps of the whole module: composition (the assigner's), review (the runner's own verb) and the +// approval, asked last (§22, §26). +var moduleSteps = []string{seat.StepResultStepComposition, seat.StepResultStepReview, seat.StepResultStepApproval} + +// kindImage is the form of an acting rule's `on` naming an artifact kind (§324). +const kindImage = "kind: image" + +// AskID is a step's ask id, the same for the same pipeline, artifact and step, so a start asked again asks nothing new. +func AskID(pipeline, artifact, step string) string { + sum := sha256.Sum256([]byte(pipeline + "\n" + artifact + "\n" + step)) + return "step-" + hex.EncodeToString(sum[:12]) +} + +// Place places a pipeline's steps (design 47 G §21, §23, §323; A-91): only the steps the module's registration policy +// names, each artifact's lint, unit and integration steps beside the module's composition, review and approval. A step the +// policy does not name is no step. A `bypass` step is placed bypassed and asked of no one; a build step is placed running, +// asked of the build seat the artifact's language names in the catalog half (§323), or, where no seat names it, ended +// could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats' +// work queues, one per running step. +func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) { + entry := entries[p.Module] + arts := append([]catalog.Artifact(nil), entry.Artifacts...) + sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref }) + var steps []seat.StepResult + var asks []Asked + // The build step is every pipeline's first, required whatever the policy names, asked of the seat mesh-builds for the + // module's own entry only (A-99, A-97): a group's other members are built by their own pipelines. + build := seat.StepResult{Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Fingerprint: p.Fingerprint, + Seat: builds.Seat, Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: AskID(p.Pipeline, "", seat.StepResultStepBuild)} + steps = append(steps, build) + asks = append(asks, Asked{Seat: builds.Seat, ID: build.Ask, Ask: builds.StepRun{Pipeline: p.Pipeline, Step: builds.StepRunStepBuild, Branch: p.Branch, + Revision: p.Revision, Source: p.Source, Entries: []builds.BuildEntry{{Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint}}}}) + for _, a := range arts { + name := a.Ref + if _, after, ok := strings.Cut(a.Ref, ":"); ok { + name = after + } + for _, step := range buildSteps { + status := Status(reg, name, step) + if status == "" { + continue + } + s := seat.StepResult{Step: step, Artifact: a.Ref, Status: status, Fingerprint: p.Fingerprint} + if status == seat.StepResultStatusBypass { + s.Outcome, s.Why = seat.StepResultOutcomeBypassed, "bypassed by the module's policy" + steps = append(steps, s) + continue + } + on := a.Language + if on == "" && a.Kind == "image" { + on = kindImage + } + bs, err := BuildSeat(entries, on) + if err != nil { + s.Outcome, s.Started, s.Ended, s.Why = seat.StepResultOutcomeCouldNotRun, &now, &now, err.Error() + steps = append(steps, s) + continue + } + s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now + s.Ask = AskID(p.Pipeline, a.Ref, step) + steps = append(steps, s) + ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)} + if a.Package != nil { + ask.Package = a.Package.Name + } + asks = append(asks, ask) + } + } + for _, step := range moduleSteps { + status := Status(reg, "", step) + if status == "" { + continue + } + s := seat.StepResult{Step: step, Status: status, Outcome: seat.StepResultOutcomeWaiting} + if status == seat.StepResultStatusBypass { + s.Outcome, s.Why = seat.StepResultOutcomeBypassed, "bypassed by the module's policy" + } + // Composition is the assigner's, an accepted kind step of the seat mesh-wants (§22): it is asked with the others, + // running at once, and its end is heard from that seat as the build seats' are. Review and approval wait. + if step == seat.StepResultStepComposition && status != seat.StepResultStatusBypass { + s.Seat, s.Outcome, s.Started, s.Fingerprint = wants.Seat, seat.StepResultOutcomeRunning, &now, p.Fingerprint + s.Ask = AskID(p.Pipeline, "", step) + asks = append(asks, Asked{Seat: wants.Seat, ID: s.Ask, Ask: wants.StepRun{Branch: p.Branch, Commit: p.Commit, Fingerprint: p.Fingerprint, + Module: p.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: wants.StepRunStepComposition}}) + } + steps = append(steps, s) + } + return steps, asks +} + +// Asked is one ask to put on a build seat's work queue. +type Asked struct { + Seat string + ID string + Ask any + // Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results + // are the ask's (§25, §416). + Package string +} + +func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk { + fetch := nodebuildgo.Fetch{Repository: p.Repository, Commit: p.Commit, Dir: e.Path} + return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: a.Language, At: &now, + Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: p.Commit, Fetch: fetch, Fingerprint: p.Fingerprint, + Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} +} + +// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. +func askAll(asker StepAsker, arm Arm, key string, asks []Asked) error { + var first error + for _, a := range asks { + accept, ok := acceptStep[a.Seat] + err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat) + if ok { + // A build seat's holder says ask-taken: the ask's bound is armed before it is put (§535, §647). + if arm != nil && BoundSeat(a.Seat) { + if aerr := arm(a.ID); aerr != nil { + err = fmt.Errorf("its bound is not armed: %w", aerr) + ok = false + } + } + } + if ok { + err = asker.AskStep(accept, a.ID, a.Ask) + } + if err != nil && first == nil { + first = fmt.Errorf("the step %s of %s is not asked of %s: %w", a.ID, key, a.Seat, err) + } + } + return first +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go new file mode 100644 index 00000000..6ae3bc11 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -0,0 +1,268 @@ +package pipeline + +import ( + "encoding/json" + "strings" + "testing" + "time" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" +) + +func stepSeatOf(t *testing.T, name string) StepSeat { + t.Helper() + for _, bs := range StepSeats { + if bs.Seat == name { + return bs + } + } + t.Fatalf("no step seat %s", name) + return StepSeat{} +} + +type stepAsks struct { + at []string + ids []string + asks []any +} + +func (s *stepAsks) AskStep(accept, id string, body any) error { + s.at, s.ids, s.asks = append(s.at, accept), append(s.ids, id), append(s.asks, body) + return nil +} + +func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) { + reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{ + Steps: catalog.StepStatuses{Lint: "required", Unit: "optional", Approval: "required"}, + Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}}, + }} + return reg, map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, + {Ref: "app:web", Language: "rust"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + } +} + +// The runner's asking half (design 47 G §22, §24, §323; A-91): a pipeline's steps are the ones the policy names, a bypassed +// step asked of no one, an artifact no build seat names ended could not run naming it, and each running step asked of its +// build seat's accepted kind step with the pipeline's commit and fingerprint; a step the policy does not name is no step. +func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { + reg, entries := placeFixture() + store := created{} + asks := &stepAsks{} + s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner", + Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} + if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + var p seat.Pipeline + if err := json.Unmarshal(store["main.4.app"], &p); err != nil { + t.Fatal(err) + } + var got []string + for _, st := range p.Steps { + got = append(got, st.Artifact+" "+st.Step+" "+st.Outcome) + } + want := []string{" build running", "app:program lint running", "app:program unit running", "app:tools lint bypassed", "app:tools unit running", + "app:web lint could not run", "app:web unit could not run", " approval waiting"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("steps = %q, want %q (integration and composition are named by no policy: no step)", got, want) + } + if !strings.Contains(p.Steps[5].Why, `"rust"`) { + t.Errorf("the step of an artifact no seat builds says %q, want it to name the language", p.Steps[5].Why) + } + if len(asks.asks) != 4 { + t.Fatalf("asked %d steps, want the build and the three running ones", len(asks.asks)) + } + // A-99: the build is every pipeline's first step, required whatever the policy names, asked of mesh-builds for the + // module's own entry only. + b, ok := asks.asks[0].(builds.StepRun) + if !ok || asks.at[0] != builds.AcceptSubjectStep || p.Steps[0].Status != seat.StepResultStatusRequired || asks.ids[0] != p.Steps[0].Ask || + len(b.Entries) != 1 || b.Entries[0] != (builds.BuildEntry{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}) || b.Pipeline != "main.4.app" { + t.Errorf("the build ask = %+v at %s", asks.asks[0], asks.at[0]) + } + a := asks.asks[1].(nodebuildgo.StepAsk) + if asks.at[1] != nodebuildgo.AcceptSubjectCheckStep || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID || + a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4.app" || a.Step.Artifact != "app:program" || + a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 { + t.Errorf("the ask = %+v at %s", a, asks.at[1]) + } + // A second start of the revision asks nothing. + if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Entries: entries}); ok || err != nil || len(asks.asks) != 4 { + t.Errorf("second start = %v, %v, %d asks", ok, err, len(asks.asks)) + } +} + +// A pipeline whose other steps are all bypassed still has its build, required whatever the policy names, so its approval is +// not due at its start (A-99, §26). +func TestTheBuildHoldsTheApprovalWhateverThePolicyNames(t *testing.T) { + reg, entries := placeFixture() + reg.Policy = catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "bypass", Approval: "required"}} + store := created{} + s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: &stepAsks{}, + Registration: func(string) (catalog.Registration, bool) { return reg, true }} + if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); err != nil { + t.Fatal(err) + } + var p seat.Pipeline + _ = json.Unmarshal(store["main.4.app"], &p) + if p.Steps[0].Step != seat.StepResultStepBuild || p.Steps[0].Status != seat.StepResultStatusRequired || approvalDue(p) != -1 { + t.Errorf("steps = %+v; want the build first, required, and no approval due", p.Steps) + } +} + +// The seat mesh-builds' end of the build step is taken as the others are, held to its one entry's module, commit and +// fingerprint; an end naming two entries is refused (A-97, A-99). +func TestTheBuildStepsEndIsTakenOnItsOneEntry(t *testing.T) { + bs := stepSeatOf(t, builds.Seat) + h, store, _, _, _, _ := fixture() + e := store[pkey] + e.p.Steps = append([]seat.StepResult{{Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, + Ask: "b1", Seat: builds.Seat}}, e.p.Steps...) + store[pkey] = e + raw := `{"id":"b1","pipeline":"main.5.app","step":"build","node":"n1","outcome":"passed","entries":[{"module":"app","commit":"c1","fingerprint":"src2:aa"}]}` + ev, err := bs.Decode([]byte(raw)) + if err != nil { + t.Fatal(err) + } + p, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev) + if err != nil || p.Steps[0].Outcome != seat.StepResultOutcomePassed { + t.Fatalf("the build end: %v, step %+v", err, p.Steps[0]) + } + if _, err := bs.Decode([]byte(`{"id":"b1","entries":[{"module":"a","fingerprint":"x"},{"module":"b","fingerprint":"y"}]}`)); err == nil { + t.Error("an end of two entries is refused") + } +} + +// Composition is asked of the seat mesh-wants with the other steps, running at once, and its end is heard from that seat +// (design 47 G §22, §24): a pipeline whose policy names it places it running with its own ask, a bypassed one asks no one. +func TestCompositionIsAskedOfTheAssignerAndItsEndIsTaken(t *testing.T) { + reg, entries := placeFixture() + reg.Policy = catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "bypass", Composition: "required", Approval: "required"}} + p := seat.Pipeline{Pipeline: "main.4.app", Branch: "main", Module: "app", Commit: "c1", Fingerprint: "src2:aa", Revision: 7, Source: 4} + steps, asked := Place(p, reg, entries, time.Unix(1, 0), "the-runner") + var comp seat.StepResult + for _, s := range steps { + if s.Step == seat.StepResultStepComposition { + comp = s + } + } + if comp.Outcome != seat.StepResultOutcomeRunning || comp.Seat != wants.Seat || comp.Ask == "" { + t.Fatalf("composition = %+v, want running, asked of mesh-wants", comp) + } + var got *Asked + for i := range asked { + if asked[i].Seat == wants.Seat { + got = &asked[i] + } + } + r, ok := got.Ask.(wants.StepRun) + if got == nil || !ok || got.ID != comp.Ask || acceptStep[wants.Seat] != wants.AcceptSubjectStep || r.Step != "composition" || + r.Commit != "c1" || r.Fingerprint != "src2:aa" || r.Pipeline != "main.4.app" || r.Revision != 7 || r.Source != 4 || r.Module != "app" { + t.Fatalf("the composition ask = %+v", got) + } + // Its end, from the seat's own subject, is taken like the others, a node's warning becoming the step's why. + bs := stepSeatOf(t, wants.Seat) + ev, err := bs.Decode([]byte(`{"id":"` + comp.Ask + `","pipeline":"main.5.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed","warnings":["n2 asleep"]}`)) + if err != nil || ev.Said != "n2 asleep" || ev.Outcome != "passed" || ev.Module != "app" { + t.Fatalf("decoded end = %+v, %v", ev, err) + } +} + +// A runner restarted after it wrote a pipeline and before its asks reached the queues asks them again when the branch's +// view holds its half: each step placed and still running, of a running pipeline of that branch, under the id it was +// placed under; an ended step, an ended pipeline and another branch's pipeline are asked of no one (design 47 G §88). +func TestResumeAsksTheStepsPlacedAndNotEnded(t *testing.T) { + reg, entries := placeFixture() + placed := func(branch string, source int64) seat.Pipeline { + p := seat.Pipeline{Pipeline: Key(branch, source, "app"), Branch: branch, Source: source, Module: "app", Commit: "c1", Fingerprint: "src2:aa", + Repository: "o/app", Outcome: seat.PipelineOutcomeRunning} + p.Steps, _ = Place(p, reg, entries, time.Unix(1, 0), "the-runner") + return p + } + main4, main5, beta6 := placed("main", 4), placed("main", 5), placed("beta", 6) + main5.Steps[1].Outcome = seat.StepResultOutcomePassed // the build of main.5 ended + ended := placed("main", 3) + ended.Outcome = seat.PipelineOutcomePassed + store := memStore{} + for _, p := range []seat.Pipeline{main4, main5, beta6, ended} { + store[p.Pipeline] = struct { + p seat.Pipeline + rev uint64 + }{p, 1} + } + asks := &stepAsks{} + s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} + n, err := s.Resume(store, "main", Half{Entries: entries}) + // main.4 has the build and three running steps of the fixture; main.5 one fewer, its build ended. + if err != nil || n != 7 || len(asks.asks) != 7 { + t.Fatalf("resume asked %d (%d put), %v; want 7", n, len(asks.asks), err) + } + for _, id := range asks.ids { + found := false + for _, p := range []seat.Pipeline{main4, main5} { + for _, st := range p.Steps { + found = found || (st.Ask == id && st.Outcome == seat.StepResultOutcomeRunning) + } + } + if !found { + t.Errorf("ask %s is of no running step of the branch's running pipelines", id) + } + } +} + +// A step ask carries the kept results of its artifact's package and the toolchain of its language (design 47 G §25, §416, +// A-104): the results whose package is the artifact's package or lies beneath it, read once for all the asks; a failed +// read asks the steps with none. +func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) { + reg, entries := placeFixture() + e := entries["app"] + e.Artifacts[0].Package = &catalog.ArtifactPackage{Ecosystem: "go", Name: "git.example/o/app"} + entries["app"] = e + store, asks := created{}, &stepAsks{} + reads := 0 + s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner", + Registration: func(string) (catalog.Registration, bool) { return reg, true }, + Kept: func(pkgs []string) (map[string][]nodebuildgo.KeptResult, error) { + reads++ + if len(pkgs) != 1 || pkgs[0] != "git.example/o/app" { + t.Errorf("kept read for %v, want the one package of the entry", pkgs) + } + return map[string][]nodebuildgo.KeptResult{pkgs[0]: {{Digest: "sha256:aa", Level: "unit", Package: "git.example/o/app/sub"}}}, nil + }, + Toolchain: func(l string) (*nodebuildgo.ToolchainUsed, error) { + return &nodebuildgo.ToolchainUsed{Language: l, Version: "1.26"}, nil + }} + if _, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Entries: entries}); err != nil { + t.Fatal(err) + } + var program, tools *nodebuildgo.StepRun + for _, a := range asks.asks { + if ask, ok := a.(nodebuildgo.StepAsk); ok { + switch { + case ask.Step.Artifact == "app:program" && ask.Step.Step == "lint": + program = ask.Step + case ask.Step.Artifact == "app:tools" && ask.Step.Step == "unit": + tools = ask.Step + } + } + } + if reads != 1 || program == nil || len(program.Kept) != 1 || program.Kept[0].Digest != "sha256:aa" || program.Toolchain == nil || program.Toolchain.Version != "1.26" { + t.Errorf("the program's ask = %+v after %d reads", program, reads) + } + if tools == nil || len(tools.Kept) != 0 || tools.Toolchain == nil { + t.Errorf("an artifact with no package carries no kept result and its language's toolchain: %+v", tools) + } + for pkg, want := range map[string]bool{"a/b": true, "a/b/c": true, "a/bc": false, "a": false} { + if InPackage(pkg, "a/b") != want { + t.Errorf("InPackage(%s, a/b) = %v, want %v", pkg, !want, want) + } + } + if tc, err := ToolchainOf([]byte(`{"language":"go","version":"1.26.1","digests":[]}`)); err != nil || tc.Language != "go" || tc.Version != "1.26.1" { + t.Errorf("ToolchainOf = %+v, %v", tc, err) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go index 4eaa2ae2..fbe903d5 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -12,7 +12,7 @@ import ( // Status is a step's status for one artifact of a module, read from the module's registration and never from a // manifest (design 47 G §21, §23): the artifact's own status where its policy names one, else the module's. A step -// neither names is "" (design question Q-RN8, open). +// neither names is "" and is no step of the pipeline (A-91): Place places none for it. func Status(reg catalog.Registration, artifact, step string) string { if a, ok := reg.Policy.Artifacts[artifact]; ok { if s := statusOf(a, step); s != "" { @@ -81,6 +81,9 @@ type BranchView interface { View() (Half, bool) } +// readyView is a BranchView that says when it holds its first half, as meshcatalogclient.Catalog does. +type readyView interface{ Ready() <-chan struct{} } + // Half is the catalog half a branch view holds, as the runner reads it: its revision and its entries by module. type Half struct { Revision int64 @@ -95,7 +98,10 @@ type Views struct { // Open makes and starts the view of a branch; its error is said in the journal and the branch asked again at the // next registration heard. Open func(branch string) (BranchView, error) - Logf func(string, ...any) + // OnReady is told once for a branch, when its view holds a first half: the steps placed in the pipelines of that branch + // and not yet asked are asked then (§88's restart pickup). + OnReady func(branch string) + Logf func(string, ...any) mu sync.Mutex branches map[string]BranchView @@ -126,6 +132,13 @@ func (v *Views) ensure(branch string) { return } v.branches[branch] = bv + if r, ok := bv.(readyView); ok && v.OnReady != nil { + on := v.OnReady + go func() { + <-r.Ready() + on(branch) + }() + } } // Branches are the watched branches the views hold, sorted. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go index 0f150d36..7ff4f096 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations_test.go @@ -83,3 +83,33 @@ type fakeView struct { func (f fakeView) Hear(context.Context, []byte) error { f.heard[f.branch]++; return nil } func (f fakeView) View() (Half, bool) { return Half{Revision: 7}, true } + +type readyFake struct { + fakeView + ch chan struct{} +} + +func (r readyFake) Ready() <-chan struct{} { return r.ch } + +// A branch's view that holds its first half tells Views once, so the runner asks the steps placed on that branch (§88). +func TestViewsTellOnceWhenABranchHoldsItsFirstHalf(t *testing.T) { + ch := make(chan struct{}) + told := make(chan string, 4) + v := &Views{OnReady: func(b string) { told <- b }, Open: func(b string) (BranchView, error) { + return readyFake{fakeView{branch: b, heard: map[string]int{}}, ch}, nil + }} + v.Registered(catalog.Registration{Module: "n", Branches: []string{"main"}}) + v.Registered(catalog.Registration{Module: "m", Branches: []string{"main"}}) + select { + case b := <-told: + t.Fatalf("told %s before the view held a half", b) + default: + } + close(ch) + if b := <-told; b != "main" { + t.Errorf("told %q, want main", b) + } + if len(told) != 0 { + t.Error("a branch's readiness is told once") + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index c507db63..b31097da 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -11,6 +11,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-sdk/go/keys" ) @@ -34,6 +35,8 @@ type Begin struct { Group string Repository string Revision int64 + // Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them. + Entries map[string]catalog.CatalogEntry } // Key is a pipeline's key in the state pipelines, `..`, the branch escaped as a key's @@ -51,6 +54,22 @@ type Starter struct { Say func(subject string, body any) error Now func() time.Time Logf func(string, ...any) + // Registration is a module's registration, whose policy names the steps of its pipeline (§21, §23); with none, a + // pipeline starts with no step and says so. + Registration func(module string) (catalog.Registration, bool) + // Steps asks each build seat for its step, as the pipeline's asking half (§22, §24); Name is the runner's own name as an + // asker. + Steps StepAsker + Name string + // Kept reads the kept results of each named package, by the package name an artifact's catalog entry carries (§25, §416): the + // results whose package is that name or one beneath it. A step ask carries them, so a build seat runs only the packages + // with no result for their digest. + Kept func(packages []string) (map[string][]nodebuildgo.KeptResult, error) + // Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when + // none was heard, which asks the step with none. + Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error) + // Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none. + Arm Arm } // Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second @@ -67,6 +86,14 @@ func (s *Starter) Start(b Begin) (bool, error) { p := seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fingerprint: b.Fp, Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}} + var asks []Asked + if s.Registration != nil { + if reg, ok := s.Registration(b.Module); ok { + p.Steps, asks = Place(p, reg, b.Entries, now, s.Name) + } else { + s.logf("%s has no registration yet: its pipeline starts with no step", key) + } + } raw, err := json.Marshal(p) if err != nil { return false, err @@ -86,9 +113,116 @@ func (s *Starter) Start(b Begin) (bool, error) { if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil { s.logf("pipeline-started of %s is not said: %v", key, err) } + if s.Steps != nil && len(asks) > 0 { + s.carry(key, asks) + if err := askAll(s.Steps, s.Arm, key, asks); err != nil { + return true, err + } + } return true, nil } +// Resume asks again, of the build seats and the assigner, each step of branch's running pipelines that is placed and still +// running (design 47 G §88's restart pickup): a pipeline written whose asks never reached their queues (the runner stopped +// between its write and its asks, or a publish failed) is asked at the next start. Each ask carries the id its step was +// placed under, so one the seat holds already is kept once. half is the branch's catalog half, read for the steps' artifacts +// and seats as Place reads it. It says how many steps it asked. +func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { + if s.Steps == nil || s.Registration == nil { + return 0, nil + } + pkeys, err := store.Keys("*.*.*") + if err != nil { + return 0, fmt.Errorf("the pipelines are not listed: %w", err) + } + now := time.Now().UTC() + if s.Now != nil { + now = s.Now().UTC() + } + asked := 0 + var first error + for _, key := range pkeys { + p, _, err := store.Get(key) + if err != nil { + if first == nil { + first = fmt.Errorf("the pipeline %s is not read: %w", key, err) + } + continue + } + if p.Branch != branch || p.Outcome != seat.PipelineOutcomeRunning { + continue + } + reg, ok := s.Registration(p.Module) + if !ok { + continue + } + running := map[string]bool{} + for _, st := range p.Steps { + if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" { + running[st.Ask] = true + } + } + _, placed := Place(p, reg, half.Entries, now, s.Name) + var due []Asked + for _, a := range placed { + if running[a.ID] { + due = append(due, a) + } + } + if len(due) == 0 { + continue + } + s.carry(key, due) + if err := askAll(s.Steps, s.Arm, key, due); err != nil && first == nil { + first = err + } + asked += len(due) + } + return asked, first +} + +// carry puts the kept results of each ask's package and the toolchain of its language on its step ask (§25, §416, A-104). A +// read that fails asks the steps with none, so every package runs: slower, never wrong; it is said. +func (s *Starter) carry(key string, asks []Asked) { + var pkgs []string + seen := map[string]bool{} + for _, a := range asks { + if a.Package != "" && !seen[a.Package] { + seen[a.Package] = true + pkgs = append(pkgs, a.Package) + } + } + var kept map[string][]nodebuildgo.KeptResult + if s.Kept != nil && len(pkgs) > 0 { + var err error + if kept, err = s.Kept(pkgs); err != nil { + s.logf("the kept results for the steps of %s are not read, so each package runs: %v", key, err) + } + } + chains := map[string]*nodebuildgo.ToolchainUsed{} + for _, a := range asks { + ask, ok := a.Ask.(nodebuildgo.StepAsk) + if !ok || ask.Step == nil { + continue + } + if a.Package != "" { + ask.Step.Kept = kept[a.Package] + } + if s.Toolchain == nil || ask.Language == "" { + continue + } + t, done := chains[ask.Language] + if !done { + var err error + if t, err = s.Toolchain(ask.Language); err != nil { + s.logf("the toolchain of %s for the steps of %s is not read: %v", ask.Language, key, err) + } + chains[ask.Language] = t + } + ask.Step.Toolchain = t + } +} + func (s *Starter) logf(format string, args ...any) { if s.Logf != nil { s.Logf(format, args...) @@ -118,7 +252,7 @@ func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.Ou if o.Kind != outdatedKind || o.Module == "" { continue } - b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision} + b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision, Entries: half.Entries} if e, ok := half.Entries[o.Module]; ok { b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository } else { @@ -134,3 +268,35 @@ func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.Ou } return started, first } + +// TakenInSubject is where any module's take-in.taken is said, `mesh.mod..event.take-in.taken`: the runner names no +// module (§7), so the bus's grant from the manifest's `consumes` says which one it hears (§54). +const TakenInSubject = "mesh.mod.*.event.take-in.taken" + +// TakenIn starts, for a take-in.taken, one pipeline per entry of the source revision, keyed by its watched branch, that +// source revision and the module (design 47 G §44, §54, §476; ADR 0666 §13). A hand drop takes nothing in and starts +// nothing; a second start of an entry, a package-published having started it first, writes nothing (§54). view is the +// catalog half of the event's branch, read only for the revision its pipelines are made from. +func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half, bool)) (int, error) { + if ev.Dropped { + s.logf("the take-in %s was dropped by %s: it starts no pipeline", ev.Key, ev.By) + return 0, nil + } + if ev.Branch == "" || ev.Source <= 0 { + return 0, fmt.Errorf("a take-in.taken names its watched branch and source revision: key %q branch %q source %d", ev.Key, ev.Branch, ev.Source) + } + half, _ := view(ev.Branch) + started := 0 + var first error + for _, e := range ev.Entries { + ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint, + Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries}) + if err != nil && first == nil { + first = err + } + if ok { + started++ + } + } + return started, first +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 5b657f5b..32843100 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -85,3 +85,31 @@ func TestStarts(t *testing.T) { type failing struct{} func (failing) Create(string, []byte) error { return errors.New("bus away") } + +// A take-in.taken starts one pipeline per entry of its source revision (design 47 G §44, §54, §476); a hand drop starts +// none, and a publish that started an entry first leaves the take-in's start of it writing nothing. +func TestTakenInStarts(t *testing.T) { + store := created{} + var said []string + s := &Starter{Pipelines: store, Say: func(subject string, body any) error { said = append(said, subject); return nil }} + view := func(string) (Half, bool) { return Half{Revision: 12}, true } + ev := catalog.TakeInTaken{Kind: "taken", Branch: "main", Source: 7, Group: "g1", Entries: []catalog.TakeInEntry{ + {Module: "a", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/a"}, {Module: "b", Commit: "c2", Fingerprint: "src2:bb"}}} + if _, err := s.Start(Begin{Branch: "main", Source: 7, Module: "b", Commit: "c2"}); err != nil { + t.Fatal(err) + } + n, err := s.TakenIn(ev, view) + if err != nil || n != 1 || len(said) != 2 { + t.Fatalf("a take-in started %d (%v), said %v: want a only, b being started already", n, err, said) + } + var p seat.Pipeline + if err := json.Unmarshal(store["main.7.a"], &p); err != nil || p.Commit != "c1" || p.Group != "g1" || p.Revision != 12 || p.Fingerprint != "src2:aa" { + t.Errorf("the pipeline main.7.a = %+v (%v)", p, err) + } + if n, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Dropped: true, By: "op", Branch: "main", Source: 8, Entries: ev.Entries}, view); n != 0 || err != nil { + t.Errorf("a hand drop started %d (%v)", n, err) + } + if _, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Branch: "main"}, view); err == nil { + t.Error("a take-in naming no source revision is refused") + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go new file mode 100644 index 00000000..21063641 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -0,0 +1,313 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + nodebuildimage "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage" + nodebuildnode "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// StepSeat is a seat whose holder runs the lint and test steps (design 47 G §22, §24): where the runner hears its +// step-ended, on the subject of the node that ran it. +type StepSeat struct { + Seat string + // Prefix is the subject before . + Prefix string + // Decode reads the seat's step-ended body; nil is the build seats' one form. + Decode func([]byte) (StepEnded, error) +} + +// StepSeats are the three build seats, from their contracts. +var StepSeats = []StepSeat{ + {Seat: nodebuildgo.Seat, Prefix: strings.TrimSuffix(nodebuildgo.SubjectStepEndedAll, "*")}, + {Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")}, + {Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")}, + // The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99). + {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded}, + // The seat mesh-wants answers the composition step (§22, §24). + {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded}, +} + +// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a +// warning of a node asleep (a pass with a warning, ADR 0593 §9) is the step's `why` when the end says nothing else. +func decodeWantsEnded(raw []byte) (StepEnded, error) { + var w wants.StepEnded + if err := json.Unmarshal(raw, &w); err != nil { + return StepEnded{}, err + } + said := w.Said + if said == "" && len(w.Warnings) > 0 { + said = strings.Join(w.Warnings, "; ") + } + return StepEnded{V: w.V, ID: w.ID, Pipeline: w.Pipeline, Step: w.Step, Module: w.Module, Commit: w.Commit, Fingerprint: w.Fingerprint, + Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil +} + +// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one +// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is +// refused (A-99). +func decodeBuildEnded(raw []byte) (StepEnded, error) { + var b builds.StepEnded + if err := json.Unmarshal(raw, &b); err != nil { + return StepEnded{}, err + } + if len(b.Entries) != 1 { + return StepEnded{}, fmt.Errorf("%w: ask %s ended a build of %d entries, asked of one", ErrRefused, b.ID, len(b.Entries)) + } + e := b.Entries[0] + return StepEnded{V: b.V, ID: b.ID, Pipeline: b.Pipeline, Step: b.Step, Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, + Node: b.Node, Outcome: b.Outcome, Said: b.Said, Rerun: b.Rerun, By: b.By, Started: b.Started, Ended: b.Ended}, nil +} + +// StepEnded is a build seat's step-ended. The three seats' bodies are one form, the part build-step's, held equal by +// their definitions, so the runner reads each in this one type. +type StepEnded = nodebuildgo.StepEnded + +// ErrRefused is a step-ended the runner does not take: not the node's holder's subject, or naming an ask, a commit or a +// fingerprint the runner did not ask (ADR 0666 §21; design 47 G §24). It is said, never taken. +var ErrRefused = errors.New("a step's end the runner does not take") + +// ErrChanged is an Updater's answer to an Update whose revision is no longer the key's: another write came between. +var ErrChanged = errors.New("the pipeline was written since it was read") + +// Updater is the state pipelines read and written by compare-and-set on the revision read (§27). +type Updater interface { + Get(key string) (seat.Pipeline, uint64, error) + Update(key string, p seat.Pipeline, revision uint64) (uint64, error) + // Keys are the keys matching a filter, `*` one key part. + Keys(filter string) ([]string, error) +} + +// Hearer takes the steps' ends into their pipelines and ends a pipeline whose every step ended (design 47 G §24, §27). +type Hearer struct { + Store Updater + // Say says one of the runner's events of a pipeline, on its subject; Log appends to the step log. + Say func(subject string, body any) error + Log func(seat.LogEntry) error + // Keep writes one package's passing result into the state kept, under its key (§27): only from a step's end the + // runner took, so a kept result is its seat holder's alone. + Keep func(key string, k seat.KeptResult) error + // Output keeps a step's output under key in the objects step-outputs. + Output func(key string, data []byte) error + // Asker publishes an approval step's ask (§37); Name is the runner's own name, the asker a warrant names. + Asker Asker + Name string + // Where reads the router's record of one of the runner's asks, as operatorchannelclient.Client.Where does: its state and + // its warrant (design 47 G A-89, §700). + Where func(ask string) (string, *oc.Warrant, error) + Logf func(string, ...any) + Now func() time.Time +} + +func (h *Hearer) now() time.Time { + if h.Now != nil { + return h.Now().UTC() + } + return time.Now().UTC() +} + +// OnStepEnded takes the step-ended heard on subject, from seat's holders. It is refused, and said, when the subject's +// node is not the one the body names, when no pipeline the runner keeps names the ask, or when the ask's step names +// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended. to that node's +// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again). +func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) { + node := strings.TrimPrefix(subject, bs.Prefix) + if node == subject || node == "" || strings.Contains(node, ".") { + return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat) + } + if ev.Node != node { + return seat.Pipeline{}, fmt.Errorf("%w: ask %s ended on %s's subject names node %q", ErrRefused, ev.ID, node, ev.Node) + } + for { + p, rev, err := h.Store.Get(ev.Pipeline) + if err != nil { + return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err) + } + i := stepOfAsk(p, ev.ID) + switch { + case i < 0: + return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID) + case p.Steps[i].Seat != bs.Seat: + return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat) + case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step: + return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step, + ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint) + case p.Steps[i].Outcome != seat.StepResultOutcomeRunning: + return p, nil + case !endOutcome(ev.Outcome): + return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome) + } + now := h.now() + s := &p.Steps[i] + s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now + if ev.Tests != nil { + s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} + } + if ev.Output != "" && h.Output != nil { + name := seat.StepOutputsByPipelineStepArtifactKey(keys.EscapeName(ev.Pipeline), s.Step, keys.EscapeName(s.Artifact)) + if err := h.Output(name, []byte(ev.Output)); err != nil { + return p, fmt.Errorf("the output of ask %s cannot be kept: %w", ev.ID, err) + } + s.Output = name + } + from := now + if ev.Ended != nil { + from = ev.Ended.UTC() + } + asked, err := askApproval(ev.Pipeline, &p, from, h.Asker) + if err != nil { + return p, err + } + ended := endPipeline(&p, now) + if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err) + } + if h.Keep != nil { + for _, k := range ev.Kept { + kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline} + if k.Toolchain != nil { + kept.Toolchain = &seat.ToolchainUsed{Language: k.Toolchain.Language, Recipe: k.Toolchain.Recipe, Version: k.Toolchain.Version} + } + if kept.Pipeline == "" { + kept.Pipeline = ev.Pipeline + } + if err := h.Keep(seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)), kept); err != nil { + h.logf("the passing result of %s (%s) at %s is not kept, so it runs again next time: %v", k.Package, k.Level, k.Digest, err) + } + } + } + h.logEnd(ev.Pipeline, *s, now) + if err := h.say(seat.EventStepEnded, p, s); err != nil { + return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err) + } + return p, h.said(p, asked, ended) + } +} + +// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall. +func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error { + if asked { + if err := h.say(seat.EventWaitHeld, p, nil); err != nil { + return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err) + } + } + if ended { + if err := h.say(seat.EventPipelineEnded, p, nil); err != nil { + return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err) + } + } + return nil +} + +func (h *Hearer) logEnd(key string, s seat.StepResult, at time.Time) { + if h.Log == nil { + return + } + if err := h.Log(seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome, + Output: s.Output, At: at}); err != nil { + h.logf("the step log misses the end of %s of %s: %v", s.Step, key, err) + } +} + +func (h *Hearer) logf(format string, args ...any) { + if h.Logf != nil { + h.Logf(format, args...) + } +} + +// say says one event of p on the runner's subject for it; step, when not nil, is the result a step-ended carries. +func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error { + last := p.Commit + if p.Group != "" { + last = p.Group + } + subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last) + at := h.now() + switch event { + case seat.EventStepEnded, seat.EventStepStarted: + return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, + Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step}) + case seat.EventPipelineEnded: + return h.Say(subject, pipelineEnded(p)) + default: + return h.Say(subject, seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint, + Group: p.Group, Revision: p.Revision, Source: p.Source, Step: seat.StepResultStepApproval, On: "operator", Since: at}) + } +} + +func pipelineEnded(p seat.Pipeline) seat.PipelineEnded { + return seat.PipelineEnded{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint, + Group: p.Group, Revision: p.Revision, Source: p.Source, Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started, + Ended: p.Ended, EndCount: p.EndCount, Steps: p.Steps} +} + +func endOutcome(o string) bool { + switch o { + case nodebuildgo.StepEndedOutcomePassed, nodebuildgo.StepEndedOutcomeFailed, nodebuildgo.StepEndedOutcomeNothingToRun, + nodebuildgo.StepEndedOutcomeCouldNotRun: + return true + } + return false +} + +func stepOfAsk(p seat.Pipeline, id string) int { + for i, s := range p.Steps { + if id != "" && s.Ask == id { + return i + } + } + return -1 +} + +// endPipeline ends a pipeline whose every step ended: passed when every required step passed or was carried, else +// failed, naming each required step that held it (ADR 0666 §7, §9; design 47 G §24, §26). A failed optional step, and an optional step +// still waiting, hold nothing. An approval step still waiting when a required step did not pass is `not asked` (§26). It says whether the +// pipeline ended now, its end count one higher. +func endPipeline(p *seat.Pipeline, now time.Time) bool { + if p.Outcome != seat.PipelineOutcomeRunning { + return false + } + var held []string + notAsked := -1 + for i, s := range p.Steps { + switch { + case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting: + notAsked = i + continue + case s.Outcome == seat.StepResultOutcomeWaiting && s.Status != seat.StepResultStatusRequired: + // An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the + // pipeline's end (A-104). + continue + case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning: + return false + } + if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried { + held = append(held, s.Artifact+" "+s.Step+" "+s.Outcome) + } + } + if notAsked >= 0 { + if len(held) == 0 { + return false + } + p.Steps[notAsked].Outcome = seat.StepResultOutcomeNotAsked + } + p.Ended = &now + p.EndCount++ + p.Outcome, p.Why = seat.PipelineOutcomePassed, "" + if len(held) > 0 { + p.Outcome, p.Why = seat.PipelineOutcomeFailed, "required steps not passed: "+strings.Join(held, ", ") + } + return true +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go new file mode 100644 index 00000000..fec31c8c --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go @@ -0,0 +1,350 @@ +package pipeline + +import ( + "errors" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +type memStore map[string]struct { + p seat.Pipeline + rev uint64 +} + +func (m memStore) Get(key string) (seat.Pipeline, uint64, error) { + e, ok := m[key] + if !ok { + return seat.Pipeline{}, 0, errors.New("no such key") + } + // A read is a copy, as one from the bus is: a change not written is not seen by the next read. + e.p.Steps = append([]seat.StepResult(nil), e.p.Steps...) + return e.p, e.rev, nil +} + +func (m memStore) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) { + if m[key].rev != rev { + return 0, ErrChanged + } + m[key] = struct { + p seat.Pipeline + rev uint64 + }{p, rev + 1} + return rev + 1, nil +} + +func (m memStore) Keys(filter string) ([]string, error) { + var out []string + for k := range m { + if ok, _ := matchKey(filter, k); ok { + out = append(out, k) + } + } + return out, nil +} + +func matchKey(filter, key string) (bool, error) { + f, k := strings.Split(filter, "."), strings.Split(key, ".") + if len(f) != len(k) { + return false, nil + } + for i := range f { + if f[i] != "*" && f[i] != k[i] { + return false, nil + } + } + return true, nil +} + +type askRecorder struct{ asked []oc.Ask } + +func (a *askRecorder) Ask(x oc.Ask) error { a.asked = append(a.asked, x); return nil } + +const pkey = "main.5.app" + +// fixture is a pipeline of app with a required lint asked of node-build-go, an optional unit asked of the same seat and an +// approval step waiting, and a hearer over it. +func fixture() (*Hearer, memStore, *[]string, *askRecorder, map[string]seat.KeptResult, map[string]string) { + store := memStore{pkey: {p: seat.Pipeline{V: 1, Pipeline: pkey, Branch: "main", Source: 5, Module: "app", Commit: "c1", Fingerprint: "src2:aa", + Outcome: seat.PipelineOutcomeRunning, Steps: []seat.StepResult{ + {Artifact: "program", Step: seat.StepResultStepLint, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "a1", Seat: nodebuildgo.Seat}, + {Artifact: "program", Step: seat.StepResultStepUnit, Status: seat.StepResultStatusOptional, Outcome: seat.StepResultOutcomeRunning, Ask: "a2", Seat: nodebuildgo.Seat}, + {Artifact: "program", Step: seat.StepResultStepApproval, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeWaiting}, + }}, rev: 1}} + said := &[]string{} + asker := &askRecorder{} + kept := map[string]seat.KeptResult{} + outs := map[string]string{} + h := &Hearer{Store: store, Name: "mesh-pipeline-runner", Asker: asker, + Say: func(subject string, _ any) error { + *said = append(*said, subject[strings.Index(subject, ".event.")+7:]) + return nil + }, + Log: func(seat.LogEntry) error { return nil }, + Keep: func(k string, v seat.KeptResult) error { kept[k] = v; return nil }, + Output: func(k string, d []byte) error { outs[k] = string(d); return nil }, + Now: func() time.Time { return time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) }} + return h, store, said, asker, kept, outs +} + +func ended(ask, step, outcome string) StepEnded { + return StepEnded{ID: ask, Ask: ask, Node: "n1", Pipeline: pkey, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: step, Outcome: outcome} +} + +var goSeat = StepSeat{Seat: nodebuildgo.Seat, Prefix: "mesh.seat.node-build-go.event.step-ended."} + +// A step's end is taken only as the runner asked it (design 47 G §24; ADR 0666 §21). +func TestStepEndedIsTakenOnlyAsAsked(t *testing.T) { + h, store, _, _, _, _ := fixture() + other := ended("a1", "lint", "passed") + other.Fingerprint = "src2:bb" + wrongNode := ended("a1", "lint", "passed") + wrongNode.Node = "n2" + unknownPipeline := ended("a1", "lint", "passed") + unknownPipeline.Pipeline = "main.9.app" + for name, c := range map[string]struct { + subject string + ev StepEnded + }{ + "a subject of another seat": {"mesh.seat.node-build-node.event.step-ended.n1", ended("a1", "lint", "passed")}, + "a body naming another node": {goSeat.Prefix + "n1", wrongNode}, + "an unknown pipeline": {goSeat.Prefix + "n1", unknownPipeline}, + "an ask nobody made": {goSeat.Prefix + "n1", ended("zz", "lint", "passed")}, + "another fingerprint": {goSeat.Prefix + "n1", other}, + "another step": {goSeat.Prefix + "n1", ended("a1", "unit", "passed")}, + "an outcome step-ended lacks": {goSeat.Prefix + "n1", ended("a1", "lint", "waiting")}, + } { + if _, err := h.OnStepEnded(goSeat, c.subject, c.ev); !errors.Is(err, ErrRefused) { + t.Errorf("%s: err %v, want a refusal", name, err) + } + } + if store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning { + t.Error("a refused end changed the pipeline") + } + // The seat the step was asked of is the only one that ends it. + if _, err := h.OnStepEnded(StepSeat{Seat: "node-build-node", Prefix: "mesh.seat.node-build-node.event.step-ended."}, "mesh.seat.node-build-node.event.step-ended.n1", ended("a1", "lint", "passed")); !errors.Is(err, ErrRefused) { + t.Errorf("another seat's holder ended the step: %v", err) + } +} + +// A taken end is written, kept and said; the pipeline ends once every step ended (§24, §27). +func TestStepEndedEndsThePipeline(t *testing.T) { + h, store, said, _, kept, outs := fixture() + h.Asker = nil + lint := ended("a1", "lint", "passed") + lint.Output = "all clean" + unit := ended("a2", "unit", "failed") + unit.Kept = []nodebuildgo.KeptResult{{Digest: "sha256:d1", Level: "unit", Package: "app/x"}} + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", lint); err != nil { + t.Fatal(err) + } + if p := store[pkey].p; p.Outcome != seat.PipelineOutcomeRunning || p.Steps[0].Node != "n1" || p.Steps[0].Output == "" { + t.Fatalf("after the lint: %+v", p) + } + if len(outs) != 1 { + t.Errorf("outputs kept %v", outs) + } + // Approval waits with no asker, so the pipeline does not end on the last test step alone. + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", unit); err != nil { + t.Fatal(err) + } + if p := store[pkey].p; p.Outcome != seat.PipelineOutcomeRunning { + t.Fatalf("a pipeline with its approval waiting ended: %+v", p) + } + if k := kept[seat.KeptByLevelDigestKey("unit", keys.EscapeName("sha256:d1"))]; k.Package != "app/x" || k.Pipeline != pkey { + t.Errorf("kept %+v", kept) + } + // A step heard again takes nothing. + n := len(*said) + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", unit); err != nil || len(*said) != n { + t.Errorf("a second end said %v (%v)", (*said)[n:], err) + } +} + +// The approval step is asked last, only when every other required step passed, and ended by its warrant (§26, §37). +func TestApprovalIsAskedLastAndEndedByItsWarrant(t *testing.T) { + h, store, said, asker, _, _ := fixture() + var record *oc.Warrant + h.Where = func(string) (string, *oc.Warrant, error) { return "chosen", record, nil } + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ended("a1", "lint", "passed")); err != nil || len(asker.asked) != 0 { + t.Fatalf("asked %d before the unit step ended (%v)", len(asker.asked), err) + } + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ended("a2", "unit", "failed")); err != nil { + t.Fatal(err) + } + // An optional step that failed holds nothing: the approval is asked. + if len(asker.asked) != 1 || asker.asked[0].About != "app.main" { + t.Fatalf("asked %+v", asker.asked) + } + a := asker.asked[0] + if got := (*said)[len(*said)-1]; !strings.HasPrefix(got, "wait-held.app.main.c1") { + t.Errorf("the wait was said as %q", got) + } + warrant := func(option, outcome string) oc.Warrant { + return oc.Warrant{Asker: "mesh-pipeline-runner", Ask: a.ID, About: a.About, Outcome: outcome, Option: option, Level: operatorchannelclient.Approve, + AskDigest: operatorchannelclient.Digest(a), At: operatorchannelclient.FormatTime(time.Date(2026, 11, 11, 13, 0, 0, 0, time.UTC)), + By: &oc.Person{Who: operatorchannelclient.Operator, Identity: "op", Kind: "telegram", Verified: "linked"}} + } + // The event only wakes the runner (A-89): an event that says approve while the router's record holds none takes nothing. + event := warrant(OptionApprove, oc.WarrantOutcomeChosen) + if _, err := h.OnDecided(event); !errors.Is(err, ErrNoRecord) || store[pkey].p.Steps[2].Outcome != seat.StepResultOutcomeRunning { + t.Errorf("an event with no record was taken: %v", err) + } + // A record whose warrant is for another ask's digest is not taken; an event of another asker neither. + bad := warrant(OptionApprove, oc.WarrantOutcomeChosen) + bad.AskDigest = "sha256:other" + record = &bad + if _, err := h.OnDecided(event); err == nil || store[pkey].p.Steps[2].Outcome != seat.StepResultOutcomeRunning { + t.Errorf("a warrant for another ask was taken: %v", err) + } + notMine := event + notMine.Asker = "someone" + if _, err := h.OnDecided(notMine); !errors.Is(err, ErrRefused) { + t.Errorf("another asker's event: %v", err) + } + // The record says decline although the event body says approve: the record decides. + decline := warrant(OptionDecline, oc.WarrantOutcomeChosen) + record = &decline + if p, err := h.OnDecided(event); err != nil || p.Steps[2].Outcome != seat.StepResultOutcomeDeclined { + t.Fatalf("the record said decline: %+v (%v)", p.Steps[2], err) + } + // (the declined pipeline above is a fresh fixture's: approve it on a new one) + h, store, said, asker, _, _ = fixture() + h.Where = func(string) (string, *oc.Warrant, error) { return "chosen", record, nil } + for _, s := range []StepEnded{ended("a1", "lint", "passed"), ended("a2", "unit", "failed")} { + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", s); err != nil { + t.Fatal(err) + } + } + a = asker.asked[0] + approve := warrant(OptionApprove, oc.WarrantOutcomeChosen) + record = &approve + p, err := h.OnDecided(event) + if err != nil || p.Steps[2].Outcome != seat.StepResultOutcomePassed || p.Outcome != seat.PipelineOutcomePassed || p.EndCount != 1 { + t.Fatalf("approved: %+v (%v)", p, err) + } + if !strings.HasPrefix((*said)[len(*said)-1], "pipeline-ended.app.main.c1") { + t.Errorf("said %v", *said) + } + // A pipeline that ended takes no second decision. + record = &decline + if p, err := h.OnDecided(event); err != nil || p.Steps[2].Outcome != seat.StepResultOutcomePassed { + t.Errorf("a second warrant changed it: %+v (%v)", p.Steps[2], err) + } +} + +// A decline, an expiry or a replacement ends the approval step declined or superseded, and a failed required step leaves it +// not asked with the pipeline failed (§26, §37). +func TestApprovalEnds(t *testing.T) { + for name, c := range map[string]struct { + outcome, option, want string + }{ + "declined": {oc.WarrantOutcomeChosen, OptionDecline, seat.StepResultOutcomeDeclined}, + "expired": {oc.WarrantOutcomeExpired, "", seat.StepResultOutcomeDeclined}, + "refused": {oc.WarrantOutcomeRefused, "", seat.StepResultOutcomeDeclined}, + "superseded": {oc.WarrantOutcomeReplaced, "", seat.StepResultOutcomeSuperseded}, + } { + h, _, _, asker, _, _ := fixture() + for _, s := range []StepEnded{ended("a1", "lint", "passed"), ended("a2", "unit", "passed")} { + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", s); err != nil { + t.Fatal(err) + } + } + a := asker.asked[0] + rec := oc.Warrant{Asker: h.Name, Ask: a.ID, About: a.About, Outcome: c.outcome, Option: c.option, Level: operatorchannelclient.Approve, + AskDigest: operatorchannelclient.Digest(a), At: operatorchannelclient.FormatTime(time.Date(2026, 11, 11, 13, 0, 0, 0, time.UTC)), + By: &oc.Person{Who: operatorchannelclient.Operator, Identity: "op", Kind: "telegram", Verified: "linked"}} + h.Where = func(string) (string, *oc.Warrant, error) { return c.outcome, &rec, nil } + p, err := h.OnDecided(oc.Warrant{Asker: h.Name, Ask: a.ID, About: a.About}) + if err != nil || p.Steps[2].Outcome != c.want || p.Outcome != seat.PipelineOutcomeFailed { + t.Errorf("%s: step %q, pipeline %q (%v)", name, p.Steps[2].Outcome, p.Outcome, err) + } + } + h, store, _, asker, _, _ := fixture() + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ended("a1", "lint", "failed")); err != nil { + t.Fatal(err) + } + if _, err := h.OnStepEnded(goSeat, goSeat.Prefix+"n1", ended("a2", "unit", "passed")); err != nil { + t.Fatal(err) + } + p := store[pkey].p + if len(asker.asked) != 0 || p.Steps[2].Outcome != seat.StepResultOutcomeNotAsked || p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(p.Why, "lint") { + t.Errorf("asked %d; %+v", len(asker.asked), p) + } +} + +// An optional step still waiting (a code review nobody has given) does not hold a pipeline's end; a required one does +// (design 47 G A-104). +func TestAnOptionalStepStillWaitingHoldsNoEnd(t *testing.T) { + now := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) + for name, c := range map[string]struct { + review string + ends bool + }{"optional review waiting": {seat.StepResultStatusOptional, true}, "required review waiting": {seat.StepResultStatusRequired, false}} { + p := seat.Pipeline{Outcome: seat.PipelineOutcomeRunning, Steps: []seat.StepResult{ + {Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomePassed}, + {Step: seat.StepResultStepReview, Status: c.review, Outcome: seat.StepResultOutcomeWaiting}, + {Step: seat.StepResultStepApproval, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomePassed}, + }} + if got := endPipeline(&p, now); got != c.ends { + t.Errorf("%s: ended = %v, want %v", name, got, c.ends) + } + if c.ends && p.Outcome != seat.PipelineOutcomePassed { + t.Errorf("%s: outcome %s", name, p.Outcome) + } + } +} + +// An ask no holder took within its bound ends its step `could not run` naming the seat, and with the rest ended the pipeline +// failed (design 47 G §32, §535, §647, A-104); a step that ended, or ask nobody placed, ends nothing. +func TestAnAskNoHolderTookEndsItsStepNamingTheSeat(t *testing.T) { + h, store, said, _, _, _ := fixture() + e := store[pkey] + e.p.Steps[1].Outcome = seat.StepResultOutcomePassed // the unit step ended; only lint is waiting for a holder + e.p.Steps[2].Outcome = seat.StepResultOutcomeNotAsked + store[pkey] = e + if err := h.NotTaken("nobody-asked", AskBound); err != nil || store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning { + t.Fatalf("an ask no step names ended something: %v", err) + } + if err := h.NotTaken("a1", AskBound); err != nil { + t.Fatal(err) + } + p := store[pkey].p + if p.Steps[0].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(p.Steps[0].Why, nodebuildgo.Seat) || + p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(strings.Join(*said, " "), seat.EventPipelineEnded) { + t.Errorf("step %+v, pipeline %s (%s), said %v: want could not run naming %s and a failed pipeline", p.Steps[0], p.Outcome, p.Why, *said, nodebuildgo.Seat) + } + // Its end taken once: the bound's expiry heard again changes nothing. + n := len(*said) + if err := h.NotTaken("a1", AskBound); err != nil || len(*said) != n { + t.Errorf("a second expiry said %d more events: %v", len(*said)-n, err) + } +} + +// The bound is armed before the ask is put, only for the build seats whose holder says ask-taken, and an ask whose bound is +// not armed is not put (§535, §647). +func TestTheBoundIsArmedBeforeAnAskIsPut(t *testing.T) { + var order []string + asker := askerFunc(func(accept, id string, _ any) error { order = append(order, "put "+id); return nil }) + arm := Arm(func(ask string) error { order = append(order, "arm "+ask); return nil }) + asks := []Asked{{Seat: nodebuildgo.Seat, ID: "step-1", Ask: 1}, {Seat: wants.Seat, ID: "step-2", Ask: 2}} + if err := askAll(asker, arm, "k", asks); err != nil || strings.Join(order, ",") != "arm step-1,put step-1,put step-2" { + t.Errorf("order %v, %v; want the build seat's bound armed first and mesh-wants' ask with none", order, err) + } + order = nil + failing := Arm(func(string) error { return errors.New("bucket gone") }) + if err := askAll(asker, failing, "k", asks[:1]); err == nil || len(order) != 0 { + t.Errorf("an ask whose bound cannot be armed was put or said nothing: %v %v", order, err) + } +} + +type askerFunc func(accept, id string, body any) error + +func (f askerFunc) AskStep(accept, id string, body any) error { return f(accept, id, body) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/toolchain.go b/modules/mesh-delivery-pipeline/internal/pipeline/toolchain.go new file mode 100644 index 00000000..dc4607fe --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/toolchain.go @@ -0,0 +1,28 @@ +package pipeline + +import ( + "encoding/json" + "strings" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" +) + +// ToolchainOf reads a toolchain-agreed as the toolchain a step ask names: its language and version (design 47 G §416, A-104). +// The recipe and the building node's digest are the seat's to report, not the runner's to know. +func ToolchainOf(raw []byte) (*nodebuildgo.ToolchainUsed, error) { + var t builds.ToolchainAgreed + if err := json.Unmarshal(raw, &t); err != nil { + return nil, err + } + if t.Language == "" { + return nil, nil + } + return &nodebuildgo.ToolchainUsed{Language: t.Language, Version: t.Version}, nil +} + +// InPackage says whether a kept result's package is the artifact's package name or lies beneath it (A-104: the kept entries +// whose package is the artifact's package, a Go module's packages being the module path and the paths beneath it). +func InPackage(pkg, name string) bool { + return name != "" && (pkg == name || strings.HasPrefix(pkg, name+"/")) +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go new file mode 100644 index 00000000..e664c149 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -0,0 +1,336 @@ +package pipeline + +import ( + "errors" + "fmt" + "strings" + "time" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// Verbs are the runner's write verbs `review`, `rerun`, `accept` and `cancel`, answered by its own process (design 47 G §22, +// §70, §482, A-107): each writes the state pipelines by compare-and-set on the revision read and says its event as the +// hearer does for a step's end. The caller each takes is the one the bus proved. +type Verbs struct { + Hearer *Hearer + // Starter asks a step again (rerun), as it asks the steps of a start; View reads a branch's catalog half for the ask. + Starter *Starter + View func(branch string) (Half, bool) + // Acts records a hand act before the verb does it (rerun, accept, cancel); a nil Acts refuses the verb, nothing recorded. + Acts views.HandActs + // Drop drops the bound of an ask a cancelled pipeline no longer waits on. + Drop func(ask string) error +} + +func (v Verbs) now() time.Time { return v.Hearer.now() } + +func (v Verbs) record(verb, what, why string) error { + if v.Acts == nil { + return errors.New("no way to record a hand act is given: nothing was done") + } + if strings.TrimSpace(why) == "" { + return fmt.Errorf("%s says why: nothing was done", verb) + } + if err := v.Acts.RecordHandAct(views.HandAct{What: what, Why: why, Verb: seat.Seat + "." + verb}); err != nil { + return fmt.Errorf("the hand act is not recorded, so nothing was done: %w", err) + } + return nil +} + +// change writes one change of a pipeline by compare-and-set, read again when another write came between, and says what it +// became: the step-ended of the step mutate changed (its index, -1 for none), the wait of an approval asked and the +// pipeline-ended of a pipeline it ended. It returns the written pipeline and the changed step. +func (v Verbs) change(key, event string, record func() error, mutate func(p *seat.Pipeline, now time.Time) (int, error)) (seat.Pipeline, int, error) { + h := v.Hearer + recorded := false + for { + p, rev, err := h.Store.Get(key) + if err != nil { + return p, -1, fmt.Errorf("the pipeline %s is not read: %w", key, err) + } + now := h.now() + i, err := mutate(&p, now) + if err != nil { + return p, -1, err + } + // The hand act is recorded once the verb is valid and before it writes (ADR 0565 clause 11); a refused verb records none. + if record != nil && !recorded { + if err := record(); err != nil { + return p, -1, err + } + recorded = true + } + asked, err := askApproval(key, &p, now, h.Asker) + if err != nil { + return p, i, err + } + ended := endPipeline(&p, now) + if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return p, i, fmt.Errorf("pipeline %s cannot be written: %w", key, err) + } + if i >= 0 { + if event == seat.EventStepEnded { + h.logEnd(key, p.Steps[i], now) + } + if err := h.say(event, p, &p.Steps[i]); err != nil { + return p, i, fmt.Errorf("pipeline %s's %s is written but not said: %w", key, event, err) + } + } + return p, i, h.said(p, asked, ended) + } +} + +// Review takes a code review's result for a commit and fingerprint from caller, the bus's proof, refusing none for +// authorship (§482): it ends the placed review step of each running pipeline of the module at that commit and fingerprint, +// recording caller in the step's result. It answers the first such pipeline; a review no pipeline waits on is refused. +func (v Verbs) Review(caller string, in seat.ReviewInput) (seat.ReviewAnswer, error) { + if in.Outcome != seat.ReviewInputOutcomePassed && in.Outcome != seat.ReviewInputOutcomeFailed { + return seat.ReviewAnswer{}, fmt.Errorf("a review ends %q or %q, not %q", seat.ReviewInputOutcomePassed, seat.ReviewInputOutcomeFailed, in.Outcome) + } + keys, err := v.Hearer.Store.Keys("*.*.*") + if err != nil { + return seat.ReviewAnswer{}, fmt.Errorf("the pipelines are not listed: %w", err) + } + var answer *seat.ReviewAnswer + for _, key := range keys { + p, _, err := v.Hearer.Store.Get(key) + if err != nil || p.Module != in.Module || p.Commit != in.Commit || p.Fingerprint != in.Fingerprint || reviewWaiting(p) < 0 { + continue + } + written, i, err := v.change(key, seat.EventStepEnded, nil, func(p *seat.Pipeline, now time.Time) (int, error) { + i := reviewWaiting(*p) + if i < 0 { + return -1, fmt.Errorf("%s has no review step waiting any more", key) + } + s := &p.Steps[i] + s.Outcome, s.By, s.Why, s.Ended = in.Outcome, caller, in.Said, &now + return i, nil + }) + if err != nil { + return seat.ReviewAnswer{}, err + } + if answer == nil { + answer = &seat.ReviewAnswer{Pipeline: key, Result: written.Steps[i]} + } + } + if answer == nil { + return seat.ReviewAnswer{}, fmt.Errorf("no running pipeline of %s at %s (%s) waits on a review", in.Module, in.Commit, in.Fingerprint) + } + return *answer, nil +} + +func reviewWaiting(p seat.Pipeline) int { + if p.Outcome != seat.PipelineOutcomeRunning { + return -1 + } + for i, s := range p.Steps { + if s.Step == seat.StepResultStepReview && s.Outcome == seat.StepResultOutcomeWaiting { + return i + } + } + return -1 +} + +// stepOf is the index of the step named, and of its artifact when one is named. +func stepOf(p seat.Pipeline, step, artifact string) (int, error) { + found := -1 + for i, s := range p.Steps { + if s.Step != step || (artifact != "" && s.Artifact != artifact) { + continue + } + if found >= 0 { + return -1, fmt.Errorf("%s names more than one %s step: name its artifact", p.Pipeline, step) + } + found = i + } + if found < 0 { + return -1, fmt.Errorf("%s has no %s step", p.Pipeline, step) + } + return found, nil +} + +// Accept accepts a failed `optional` step, recorded in the hand-act log first with who and why (§30, §394). +func (v Verbs) Accept(caller string, in seat.AcceptInput) (seat.AcceptAnswer, error) { + p, i, err := v.change(in.Pipeline, seat.EventStepEnded, func() error { + return v.record(seat.VerbAccept, fmt.Sprintf("accepted the failed %s step of %s", in.Step, in.Pipeline), in.Why) + }, func(p *seat.Pipeline, now time.Time) (int, error) { + i, err := stepOf(*p, in.Step, in.Artifact) + if err != nil { + return -1, err + } + s := &p.Steps[i] + if s.Status != seat.StepResultStatusOptional || s.Outcome != seat.StepResultOutcomeFailed { + return -1, fmt.Errorf("only a failed optional step is accepted: the %s step of %s is %s and %s", in.Step, in.Pipeline, s.Status, s.Outcome) + } + s.Outcome, s.Accepted, s.Ended = seat.StepResultOutcomeAccepted, &seat.Accepted{At: &now, By: caller, Why: in.Why}, &now + return i, nil + }) + if err != nil { + return seat.AcceptAnswer{}, err + } + return seat.AcceptAnswer{Result: p.Steps[i]}, nil +} + +// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and the bound of each ask dropped, recorded in +// the hand-act log first (§394); it says pipeline-ended. +func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, error) { + h := v.Hearer + recorded := false + for { + p, rev, err := h.Store.Get(in.Pipeline) + if err != nil { + return seat.CancelAnswer{}, fmt.Errorf("the pipeline %s is not read: %w", in.Pipeline, err) + } + if p.Outcome != seat.PipelineOutcomeRunning { + return seat.CancelAnswer{}, fmt.Errorf("the pipeline %s is %s, not running", in.Pipeline, p.Outcome) + } + if !recorded { + if err := v.record(seat.VerbCancel, "cancelled the pipeline "+in.Pipeline, in.Why); err != nil { + return seat.CancelAnswer{}, err + } + recorded = true + } + now := h.now() + var drop []string + for i := range p.Steps { + s := &p.Steps[i] + if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + continue + } + if s.Ask != "" { + drop = append(drop, s.Ask) + } + s.Outcome, s.By, s.Why, s.Ended = seat.StepResultOutcomeCancelled, caller, in.Why, &now + } + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeCancelled, in.Why, &now + p.EndCount++ + if _, err := h.Store.Update(in.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err) + } + if v.Drop != nil { + for _, ask := range drop { + if err := v.Drop(ask); err != nil { + h.logf("the bound of ask %s of the cancelled %s is not dropped: %v", ask, in.Pipeline, err) + } + } + } + return seat.CancelAnswer{Pipeline: p}, h.said(p, false, true) + } +} + +// Rerun asks one step again, recorded in the hand-act log first (§22, §39): the step goes back to running under a new ask, +// the ask saying it is a rerun and the caller the bus proved; a failed pipeline runs again and ends once more with its end +// count one higher (§325), an approval never asked waiting again. A review waits again; an approval is asked again by +// `ask-again`. +func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error) { + if in.Step == seat.StepResultStepApproval { + return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun") + } + var ask *Asked + p, _, err := v.change(in.Pipeline, seat.EventStepStarted, func() error { + return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why) + }, func(p *seat.Pipeline, now time.Time) (int, error) { + ask = nil + switch p.Outcome { + case seat.PipelineOutcomeCancelled, seat.PipelineOutcomeSuperseded: + return -1, fmt.Errorf("the pipeline %s is %s: a step of it is not asked again", p.Pipeline, p.Outcome) + } + i, err := stepOf(*p, in.Step, in.Artifact) + if err != nil { + return -1, err + } + s := &p.Steps[i] + if s.Outcome == seat.StepResultOutcomeRunning { + return -1, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, p.Pipeline) + } + if s.Seat != "" { + if ask, err = v.askOf(*p, *s, caller, now); err != nil { + return -1, err + } + } + *s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat, + Outcome: seat.StepResultOutcomeWaiting, By: caller, Rerun: true} + if ask != nil { + s.Outcome, s.Started, s.Ask = seat.StepResultOutcomeRunning, &now, ask.ID + } + // The pipeline runs again; an approval never asked waits again, to be asked when the step passes. + if p.Outcome != seat.PipelineOutcomeRunning { + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil + } + for j := range p.Steps { + if p.Steps[j].Step == seat.StepResultStepApproval && p.Steps[j].Outcome == seat.StepResultOutcomeNotAsked { + p.Steps[j].Outcome, p.Steps[j].Why = seat.StepResultOutcomeWaiting, "" + } + } + if ask == nil { + return -1, nil + } + return i, nil + }) + if err != nil { + return seat.RerunAnswer{}, err + } + if ask != nil { + v.Starter.carry(in.Pipeline, []Asked{*ask}) + if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}); err != nil { + return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) + } + } + res, err := stepOf(p, in.Step, in.Artifact) + if err != nil { + return seat.RerunAnswer{}, err + } + r := p.Steps[res] + return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &r}, nil +} + +// askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice +// under one id is kept once), saying it is a rerun and by whom. +func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) { + if v.Starter == nil || v.Starter.Registration == nil || v.Starter.Steps == nil || v.View == nil { + return nil, errors.New("the runner cannot ask a step: it holds no registration or no ask") + } + reg, ok := v.Starter.Registration(p.Module) + if !ok { + return nil, fmt.Errorf("%s has no registration", p.Module) + } + half, ok := v.View(p.Branch) + if !ok { + return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch) + } + _, placed := Place(p, reg, half.Entries, now, v.Starter.Name) + base := AskID(p.Pipeline, s.Artifact, s.Step) + for _, a := range placed { + if a.ID != base { + continue + } + a.ID = AskID(p.Pipeline, s.Artifact, s.Step+"\nrerun\n"+now.Format(time.RFC3339Nano)) + switch b := a.Ask.(type) { + case nodebuildgo.StepAsk: + b.ID = a.ID + if b.Step != nil { + step := *b.Step + step.Rerun, step.By = true, caller + b.Step = &step + } + a.Ask = b + case builds.StepRun: + b.Rerun, b.By = true, caller + a.Ask = b + case wants.StepRun: + b.Rerun, b.By = true, caller + a.Ask = b + } + return &a, nil + } + return nil, fmt.Errorf("the %s step of %s is placed no ask now", s.Step, p.Pipeline) +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go new file mode 100644 index 00000000..77f7ea31 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -0,0 +1,172 @@ +package pipeline + +import ( + "errors" + "strings" + "testing" + "time" + + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/views" +) + +type actLog []string + +func (a *actLog) RecordHandAct(h views.HandAct) error { + *a = append(*a, h.Verb+": "+h.What+" / "+h.Why) + return nil +} + +// verbsFixture is a running pipeline of app placed from a policy with a required lint, an optional unit, an optional review +// and a required approval, in a store, with the verbs over it. +func verbsFixture() (Verbs, memStore, *[]string, *stepAsks, *actLog, *[]string) { + reg, entries := placeFixture() + reg.Policy.Steps.Review = "optional" + h, store, said, _, _, _ := fixture() + p := seat.Pipeline{Pipeline: pkey, Branch: "main", Source: 5, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app", + Outcome: seat.PipelineOutcomeRunning} + p.Steps, _ = Place(p, reg, entries, time.Unix(1, 0), "the-runner") + store[pkey] = struct { + p seat.Pipeline + rev uint64 + }{p, 1} + asks, acts, dropped := &stepAsks{}, &actLog{}, &[]string{} + s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, + Now: h.Now} + return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts, + Drop: func(ask string) error { *dropped = append(*dropped, ask); return nil }}, store, said, asks, acts, dropped +} + +func stepNamed(p seat.Pipeline, artifact, step string) seat.StepResult { + for _, s := range p.Steps { + if s.Artifact == artifact && s.Step == step { + return s + } + } + return seat.StepResult{} +} + +// A code review ends the placed review step of the pipeline at its commit and fingerprint, whoever its author, recording the +// caller the bus proved in the step's result, its step-ended and the pipeline's show; a review no pipeline waits on is refused +// (design 47 G §70, §482). +func TestReviewEndsThePlacedReviewStepWithItsProvenCaller(t *testing.T) { + v, store, said, _, _, _ := verbsFixture() + in := seat.ReviewInput{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Outcome: "passed", Said: "reads well"} + for name, bad := range map[string]seat.ReviewInput{ + "another commit": {Module: "app", Commit: "c2", Fingerprint: "src2:aa", Outcome: "passed"}, + "another fingerprint": {Module: "app", Commit: "c1", Fingerprint: "src2:bb", Outcome: "passed"}, + "an outcome review lacks": {Module: "app", Commit: "c1", Fingerprint: "src2:aa", Outcome: "waiting"}, + } { + if _, err := v.Review("agent.dev", bad); err == nil { + t.Errorf("%s: a review was taken", name) + } + } + a, err := v.Review("agent.dev", in) + if err != nil || a.Pipeline != pkey || a.Result.Outcome != "passed" || a.Result.By != "agent.dev" { + t.Fatalf("review = %+v, %v", a, err) + } + got := stepNamed(store[pkey].p, "", seat.StepResultStepReview) + if got.Outcome != "passed" || got.By != "agent.dev" || got.Why != "reads well" || got.Ended == nil { + t.Errorf("the review step = %+v", got) + } + if len(*said) != 1 || !strings.HasPrefix((*said)[0], "step-ended.") { + t.Errorf("said %v, want one step-ended", *said) + } + if _, err := v.Review("agent.dev", in); err == nil { + t.Error("a second review of an ended review step was taken") + } +} + +// accept takes only a failed optional step, writes its hand act first, and refuses without one; rerun asks the step again +// under a new ask saying it is a rerun and its caller, reopens a failed pipeline and asks nothing of a refused one; cancel ends +// a running pipeline cancelled, each step cancelled and each ask's bound dropped (design 47 G §22, §30, §325, §394). +func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testing.T) { + v, store, said, asks, acts, dropped := verbsFixture() + end := func(step, outcome string) { + p, rev, _ := store.Get(pkey) + for i := range p.Steps { + if p.Steps[i].Artifact == "app:program" && p.Steps[i].Step == step { + p.Steps[i].Outcome = outcome + } + } + store.Update(pkey, p, rev) + } + end("unit", seat.StepResultOutcomeFailed) + if _, err := v.Accept("operator", seat.AcceptInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "x"}); err == nil { + t.Error("a running required step was accepted") + } + if _, err := v.Accept("operator", seat.AcceptInput{Pipeline: pkey, Step: "unit", Artifact: "app:program"}); err == nil { + t.Error("an accept with no why was taken") + } + a, err := v.Accept("operator", seat.AcceptInput{Pipeline: pkey, Step: "unit", Artifact: "app:program", Why: "flaky"}) + if err != nil || a.Result.Outcome != "accepted" || a.Result.Accepted == nil || a.Result.Accepted.By != "operator" || a.Result.Accepted.Why != "flaky" { + t.Fatalf("accept = %+v, %v", a, err) + } + if len(*acts) != 1 || !strings.Contains((*acts)[0], "mesh-pipeline-runner.accept") || !strings.Contains((*acts)[0], "flaky") { + t.Errorf("hand acts = %v", *acts) + } + // A rerun of the lint step: a failed pipeline runs again. + end("lint", seat.StepResultOutcomeFailed) + p, rev, _ := store.Get(pkey) + p.Outcome = seat.PipelineOutcomeFailed + store.Update(pkey, p, rev) + old := stepNamed(p, "app:program", "lint").Ask + r, err := v.Rerun("seat:mesh-healers", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "retry"}) + if err != nil || r.Result == nil || r.Result.Outcome != "running" || r.Result.Ask == old || !r.Result.Rerun || r.Result.By != "seat:mesh-healers" { + t.Fatalf("rerun = %+v, %v", r.Result, err) + } + if len(asks.asks) != 1 || asks.ids[0] != r.Result.Ask { + t.Fatalf("asked %v under %v, want one ask under %s", asks.asks, asks.ids, r.Result.Ask) + } + if b, ok := asks.asks[0].(nodebuildgo.StepAsk); !ok || b.ID != r.Result.Ask || b.Step == nil || !b.Step.Rerun || b.Step.By != "seat:mesh-healers" || b.Step.Step != "lint" { + t.Errorf("the ask = %+v", asks.asks[0]) + } + if got, _, _ := store.Get(pkey); got.Outcome != seat.PipelineOutcomeRunning { + t.Errorf("the failed pipeline is %s, want running again", got.Outcome) + } + if _, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "again"}); err == nil { + t.Error("a running step was asked again") + } + if _, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "approval", Why: "again"}); err == nil { + t.Error("an approval was asked again by rerun") + } + // A cancel ends the pipeline cancelled and drops the bounds of the asks still held. + c, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "wrong branch"}) + if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) == 0 { + t.Fatalf("cancel = %+v, %v, dropped %v", c.Pipeline, err, *dropped) + } + if s := stepNamed(c.Pipeline, "app:program", "lint"); s.Outcome != "cancelled" || s.By != "operator" { + t.Errorf("the lint step = %+v", s) + } + if _, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "again"}); err == nil { + t.Error("a cancelled pipeline was cancelled again") + } + if _, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "again"}); err == nil { + t.Error("a step of a cancelled pipeline was asked again") + } + last := (*said)[len(*said)-1] + if !strings.HasPrefix(last, "pipeline-ended.") { + t.Errorf("last said %q, want pipeline-ended", last) + } + if n := len(*acts); n != 3 { + t.Errorf("%d hand acts recorded, want accept, rerun, cancel (3): %v", n, *acts) + } +} + +// A hand act the log does not take is a verb that does nothing (ADR 0565 clause 11). +func TestAVerbWhoseHandActIsNotRecordedDoesNothing(t *testing.T) { + v, store, _, _, _, _ := verbsFixture() + v.Acts = failingActs{} + if _, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "x"}); err == nil { + t.Fatal("cancel did something") + } + if p, _, _ := store.Get(pkey); p.Outcome != seat.PipelineOutcomeRunning { + t.Errorf("the pipeline is %s", p.Outcome) + } +} + +type failingActs struct{} + +func (failingActs) RecordHandAct(views.HandAct) error { return errors.New("no holder") } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index c60c345f..36894974 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -1,6 +1,7 @@ package pipeline_test import ( + "encoding/json" "os" "slices" "testing" @@ -20,8 +21,10 @@ func TestManifest(t *testing.T) { t.Fatalf("the manifest does not read: %v", err) } var states, streams, objects []string + keyTTL := map[string]bool{} for _, s := range m.State { states = append(states, s.Name) + keyTTL[s.Name] = s.KeyTTL } for _, s := range m.Streams { streams = append(streams, s.Name) @@ -47,7 +50,9 @@ func TestManifest(t *testing.T) { {"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat && equalSets(m.Claims[0].Serves, seat.Verbs())}, // §27, §85, §330, §355, §77: the old state sections. - {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "findings", "first-views"})}, + {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "bounds", "toolchains", "findings", "first-views"})}, + // §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live. + {"bounds is key-ttl", keyTTL["bounds"]}, // §27, §85: the step log on an old stream, its outputs in an old objects section. // RN24 (§383, §387): beside it, the finding stream its findings are said on. {"old streams section", equalSets(streams, []string{seat.StateStepLog, "mesh-delivery-pipeline-findings"})}, @@ -60,15 +65,21 @@ func TestManifest(t *testing.T) { {"step-outputs 1 GiB", len(m.Objects) == 1 && m.Objects[0].MaxMiB == 1024}, // §333, §54, §44, §53, §376, §24, §37, §4, §77: what the runner hears. {"consumes", equalSets(m.Consumes, []string{ - "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", + "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", - "node-build-image.step-ended", "mesh-wants.step-ended", "mesh-catalog.registration-changed", + "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked"})}, // RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and // asked again by hand (a hand act). {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", // RN3 (§53, §334): a publication's dependents behind. - "seat:mesh-catalog.outdated"})}, + "seat:mesh-catalog.outdated", + // A-74, A-89: who holds the seat operator-channel, whose state asks holds the router's record of the approval ask. + "seat:mesh-module-assigner.assignments", + // A-102 (§22, §24): the accepted kinds the runner puts its steps on. + "seat:mesh-builds.step", "seat:mesh-wants.step", "seat:node-build-go.check.step", "seat:node-build-node.check.step", + "seat:node-build-image.check.step"})}, + {"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})}, {"first-view tools", equalSets(m.Tools, []string{"pipeline_first_views", "pipeline_first_view_again"})}, // RN1/AC5 (§95, §130): the new assigner is delivered before the runner. {"program needs the assigner's program", slices.Equal(program.Needs, []string{"mesh-delivery-assigner:program"})}, @@ -80,6 +91,31 @@ func TestManifest(t *testing.T) { } } +// §610, A-75: no verb of the runner's definition names its callers, so the class's default holds for each. +func TestNoVerbNamesItsCallers(t *testing.T) { + raw, err := os.ReadFile("../../contracts/seats/mesh-pipeline-runner/seat.json") + if err != nil { + t.Fatal(err) + } + var def struct { + Verbs []struct { + Name string `json:"name"` + Callers []string `json:"callers"` + } `json:"verbs"` + } + if err := json.Unmarshal(raw, &def); err != nil { + t.Fatal(err) + } + if len(def.Verbs) == 0 { + t.Fatal("the definition holds no verb") + } + for _, v := range def.Verbs { + if len(v.Callers) != 0 { + t.Errorf("the verb %s names callers %v; from G no verb does (§610)", v.Name, v.Callers) + } + } +} + func equalSets(a, b []string) bool { x, y := slices.Clone(a), slices.Clone(b) slices.Sort(x) diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index d0055720..044ee89a 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -30,10 +30,14 @@ "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", + "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", "node-build-image.step-ended", + "node-build-go.ask-taken", + "node-build-node.ask-taken", + "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", @@ -43,7 +47,16 @@ "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", - "seat:mesh-catalog.outdated" + "seat:mesh-catalog.outdated", + "seat:mesh-module-assigner.assignments", + "seat:mesh-builds.step", + "seat:mesh-wants.step", + "seat:node-build-go.check.step", + "seat:node-build-node.check.step", + "seat:node-build-image.check.step" + ], + "uses": [ + "operator-channel" ], "state": [ { @@ -58,6 +71,15 @@ "name": "aged", "history": 1 }, + { + "name": "toolchains", + "history": 1 + }, + { + "name": "bounds", + "history": 1, + "key-ttl": true + }, { "name": "findings", "history": 1