G-24: one composition run per candidate and machine, one test run per level, every reading step timed #244

Merged
mesh-tools merged 49 commits from fix/47-G-shared-runs into main 2026-11-17 12:15:36 +00:00
90 changed files with 9952 additions and 708 deletions
@@ -0,0 +1,375 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
// A composition run's bound (design 47 G §720; A-466n, A-466p, A-466q, A-466s, A-468b, A-469b, A-469s, A-469y, A-470): a run
// has one clock, the key `bound.<run key>` of composition-runs with its own time to live composition-step-bound, written at the
// build seat's ask-taken of the run's ask and written again at each step-progress of that ask; an ask still waiting in the queue
// has no clock but `untaken.<run key>`, written at the run's start with the time to live run-untaken-bound. Either's expiry,
// heard as the bus's event, ends the run: its ask dropped through the build seat's drop, the run ended could not run naming the
// ask and the bound, so every reader ends could not run with it. While the ask waits for a build node each reader is said waiting
// by a wait-held on mesh-wants (`on: build node`), ended `ran` at ask-taken or `superseded` when the run ends untaken.
// waitOnBuildNode is what a composition run's waiting reader waits on (A-469b, A-469y).
const waitOnBuildNode = "build node"
// dropVerbKey is the Go build seat's drop, asked as a kept call (§119, A-182).
const dropVerbKey = "seat:" + nodebuildgo.Seat + "." + nodebuildgo.VerbDrop
// dropOver asks the Go build seat's drop of a run ask as a kept call, of any holder, which records it (A-469s).
func dropOver(call func(key string, body any) (json.RawMessage, error)) func(ask, why string) error {
return func(ask, why string) error {
raw, err := call(dropVerbKey, nodebuildgo.DropInput{ID: ask, Why: why, By: runAsker})
if err != nil {
return err
}
if _, said, ok := buscalls.Unwrap(raw); !ok {
return fmt.Errorf("%s answered: %s", dropVerbKey, said)
}
return nil
}
}
// waitOf is a reader's wait for a build node to take its run's ask (A-469y).
func waitOf(run meshwants.CompositionRun, reader meshwants.CompositionReader) meshwants.Wait {
w := meshwants.Wait{Step: "pipeline", Owner: meshwants.Seat, Module: reader.Module, Node: run.Machine, On: waitOnBuildNode, Fp: reader.Fingerprint,
Words: fmt.Sprintf("the composition step of %s waits for a build node to take the composition run %s of %s (ask %s)", reader.Pipeline,
runKeyOf(run.Candidate, run.Machine), run.Machine, run.Ask)}
if run.Started != nil {
w.Since = *run.Started
}
return w
}
// sayWaits says, for each reader, its wait held or ended (how: ran, superseded); a wait that cannot be said is journaled.
func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshwants.CompositionReader, how string) {
if r.say == nil {
return
}
for _, c := range readers {
w := waitOf(run, c)
event, id := meshwants.EventWaitHeld, "composition."+runKeyOf(run.Candidate, run.Machine)+"."+run.Ask+"."+c.ID+".held"
if how != "" {
w.How, event, id = how, meshwants.EventWaitEnded, strings.TrimSuffix(id, ".held")+".ended"
}
if err := r.say(event, w, id); err != nil {
r.logf("the %s of %s on the composition run of %s is not said: %v", event, c.Pipeline, run.Machine, err)
}
}
}
// untaken is whether the run of ask has not been taken yet: its untaken key stands holding that ask (A-470n).
func (r *compositionRuns) untaken(key, ask string) bool {
held, _, standing, err := r.boundOf(runUntakenPrefix + key)
return err == nil && standing && held == ask
}
// boundOf reads a bound key: the ask it holds and its revision; standing false when it is not written.
func (r *compositionRuns) boundOf(boundKey string) (string, uint64, bool, error) {
raw, rev, err := r.kv.get(boundKey)
if err != nil || raw == nil {
return "", 0, false, err
}
var v boundValue
_ = json.Unmarshal(raw, &v)
return v.Ask, rev, true, nil
}
// armBound writes the bound key prefix+key holding ask, judged from a read of the run, then reads the run again (A-470n): when
// it no longer names ask, or has ended, the bound just written is deleted at its revision, so no bound written from the earlier
// read ends the run after it, and a bound another run wrote since is kept. It answers whether the bound stands for the run.
func (r *compositionRuns) armBound(prefix, key, ask string, ttl time.Duration) (bool, error) {
rev, err := r.kv.arm(prefix+key, ask, ttl)
if err != nil {
return false, err
}
run, _, ok, err := r.read(key)
if err != nil {
return false, err
}
if ok && run.Ended == nil && run.Ask == ask {
return true, nil
}
if err := r.kv.delAt(prefix+key, rev); err != nil && !errors.Is(err, errRunChanged) {
return false, err
}
return false, nil
}
// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed holding the ask, the untaken key of that ask
// deleted, each reader's wait ended ran. An ask no running run names is another asker's, or a late signal of a run replaced or
// ended, which moves nothing and is journaled (A-470n).
func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error {
key, run, err := r.runOfAsk(e.Asker, e.Ask)
if err != nil {
return err
}
if key == "" {
r.late("ask-taken", e.Asker, e.Ask)
return nil
}
armed, err := r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound)
if err != nil {
return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err)
}
if !armed {
r.late("ask-taken", e.Asker, e.Ask)
return nil
}
if r.untaken(key, e.Ask) {
if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil {
return err
}
r.sayWaits(run, run.Readers, "ran")
}
return nil
}
// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length); a
// late one moves nothing and is journaled (A-470n).
func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error {
key, _, err := r.runOfAsk(e.Asker, e.Ask)
if err != nil {
return err
}
armed := false
if key != "" {
if armed, err = r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound); err != nil {
return err
}
}
if !armed {
r.late("step-progress", e.Asker, e.Ask)
}
return nil
}
// late journals a signal of an ask the assigner put whose run no key names running any more (A-470n).
func (r *compositionRuns) late(signal, asker, ask string) {
if asker == runAsker && ask != "" {
r.logf("the %s of the run ask %s is late: no composition run names it running, so it moves nothing (A-470n)", signal, ask)
}
}
// runOfAsk is the running run of an ask the assigner put, or none.
func (r *compositionRuns) runOfAsk(asker, ask string) (string, meshwants.CompositionRun, error) {
if asker != runAsker || ask == "" {
return "", meshwants.CompositionRun{}, nil
}
key, run, err := r.keyOfAsk(ask)
if err != nil || key == "" || run.Ended != nil {
return "", meshwants.CompositionRun{}, err
}
return key, run, nil
}
// expired ends the run whose bound key's time to live ended (§720): its ask dropped, the run ended could not run naming the
// ask and the bound. A run that ended already, or a key of none, takes nothing.
func (r *compositionRuns) expired(boundKey string) {
key, bound, setting := "", time.Duration(0), ""
switch {
case strings.HasPrefix(boundKey, runBoundPrefix):
key, bound, setting = strings.TrimPrefix(boundKey, runBoundPrefix), r.set.stepBound, "composition-step-bound"
case strings.HasPrefix(boundKey, runUntakenPrefix):
key, bound, setting = strings.TrimPrefix(boundKey, runUntakenPrefix), r.set.untaken, "run-untaken-bound"
default:
return
}
run, _, ok, err := r.read(key)
if err != nil || !ok || run.Ended != nil {
return
}
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, and
// the run is taken for bound., untaken for untaken., so an expiry heard after a bound was armed again, or of a run replaced
// since, ends nothing.
other := runUntakenPrefix
if setting == "run-untaken-bound" {
other = runBoundPrefix
}
_, _, armed, err := r.boundOf(boundKey)
if err != nil {
r.logf("the bound %s is not read again, so its expiry ends nothing: %v", boundKey, err)
return
}
held, _, standing, err := r.boundOf(other + key)
if err != nil {
r.logf("the bound %s is not read, so the expiry of %s ends nothing: %v", other+key, boundKey, err)
return
}
if armed || (standing && held == run.Ask) {
r.logf("the expiry of %s ends nothing: the bound is armed again, or the run of ask %s is not where it bounds (A-470o)", boundKey, run.Ask)
return
}
said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting)
if setting == "run-untaken-bound" {
said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting)
}
if r.drop != nil {
if err := r.drop(run.Ask, said); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
// The untaken key is gone with its expiry, so the run ended by it is said untaken here: its readers' waits end superseded.
if _, err := r.endRunOf(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, setting == "run-untaken-bound"); err != nil {
r.logf("the composition run %s past its bound is not ended: %v", key, err)
}
}
// endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted.
func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
return r.endRunOf(key, ask, outcome, said, verdict, r.untaken(key, ask))
}
// endRunOf is endRun of a run whose ask is untaken, or not, as its caller judged.
func (r *compositionRuns) endRunOf(key, ask, outcome, said string, verdict map[string]any, untaken bool) (bool, error) {
return r.endRunMoving(key, ask, outcome, said, verdict, untaken, true)
}
// endRunMoving is endRunOf, moving the readers a rerun carried or not: a drop says its readers' ends itself (A-470r).
func (r *compositionRuns) endRunMoving(key, ask, outcome, said string, verdict map[string]any, untaken, move bool) (bool, error) {
// The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own
// step still waits on it, so it alone says its own end; each mover waits for the end in the key.
if moved := r.moved.Load(); moved != nil && move {
if run, _, ok, err := r.read(key); err == nil && ok && run.Ended == nil && run.Ask == ask {
(*moved)(run)
}
}
ended, err := r.end(key, ask, outcome, said, verdict)
if err != nil || !ended {
return ended, err
}
if untaken {
if run, _, ok, err := r.read(key); err == nil && ok {
r.sayWaits(run, run.Readers, "superseded")
}
}
if err := r.clearBounds(key, ask, runBoundPrefix, runUntakenPrefix); err != nil {
r.logf("the bound keys of the ended composition run %s are not deleted: %v", key, err)
}
return true, nil
}
// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at
// its start (§719), each reader's end said from the keys alone (A-468). A reader whose step said its end already, carried into a
// rerun's run, is taken up only when it is marked `rerun` (A-470l, A-470m); each with the ask of the open run it is taken up from.
func (r *compositionRuns) readersOfOpen() ([]openReader, error) {
all, err := r.keys()
if err != nil {
return nil, err
}
seen := map[string]bool{}
var out []openReader
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err != nil || !ok || run.Ended != nil {
continue
}
for _, c := range run.Readers {
if c.Ended != nil && !c.Rerun {
continue // it ended passed already: only a reader marked rerun says its end again
}
if !seen[c.ID] {
seen[c.ID] = true
out = append(out, openReader{CompositionReader: c, ask: run.Ask})
}
}
}
return out, nil
}
// openReader is a reader taken up at the start and the ask of the open run it is taken up from.
type openReader struct {
meshwants.CompositionReader
ask string
}
// keysOf are the keys of the candidate's runs that the reader reads.
func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
all, err := r.keys()
if err != nil {
return nil, err
}
var out []string
prefix := keysPart(candidate) + "."
for _, key := range all {
if !isRunKey(key) || !strings.HasPrefix(key, prefix) {
continue
}
if run, _, ok, err := r.read(key); err == nil && ok && hasReader(run.Readers, id) {
out = append(out, key)
}
}
slices.Sort(out) // by machine, as a step names its machines
return out, nil
}
// hearRunBounds hears the build seats' ask-taken and step-progress on the assigner's durable consumer (§720, A-466q): each
// handed to the runs; one that cannot be written is given again by the bus.
func hearRunBounds(hub interface {
Handle(string, func(jetstream.Msg)) error
}, r *compositionRuns, logf func(string, ...any)) error {
if err := hub.Handle(nodebuildgo.SubjectAskTakenAll, func(m jetstream.Msg) {
var e nodebuildgo.AskTaken
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no ask-taken, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
ackOrNak(m, r.heardAskTaken(e), logf)
}); err != nil {
return err
}
return hub.Handle(nodebuildgo.SubjectStepProgressAll, func(m jetstream.Msg) {
var e nodebuildgo.StepProgress
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-progress, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
ackOrNak(m, r.heardProgress(e), logf)
})
}
func ackOrNak(m jetstream.Msg, err error, logf func(string, ...any)) {
if err != nil {
logf("%s is not taken, so the bus gives it again: %v", m.Subject(), err)
_ = m.Nak()
return
}
_ = m.Ack()
}
// takeUp says, at the assigner's start, the end of each reader of a run with no `ended` once its runs end (§719, A-468): the
// step's goroutine is gone, so its end is judged from the candidate's keys naming it.
func (c *compositionSteps) takeUp(ctx context.Context, r *compositionRuns) {
readers, err := r.readersOfOpen()
if err != nil {
c.logf("composition-runs: the runs standing at the start are not read, so their readers are not taken up: %v", err)
return
}
for _, reader := range readers {
c.logf("the composition step %s of %s is taken up again by its runs (ask %s)", reader.Pipeline, reader.Module, reader.ID)
again := "" // a reader whose step said its end already says it again under the open run's ask (A-470q)
if reader.Ended != nil {
again = reader.ask
}
go c.say1(ctx, reader.ID, stepRunOf(reader.CompositionReader), c.resume, again)
}
}
@@ -10,7 +10,6 @@ import (
"errors"
"fmt"
"slices"
"sync"
"time"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
@@ -18,8 +17,8 @@ import (
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input,
// asked of the machine's node-engine by the seat's verb compose-candidate.
// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that
// machine's input, asked of the machine's node-engine by the seat's verb compose-candidate when the run starts.
// composeCandidateKey is the seat verb's call key, asked of one node.
const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
@@ -80,37 +79,14 @@ type compositionInputs struct {
bound time.Duration
}
// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in
// name order: one away (away) is not asked; a node that is silent, unready or answers no input has Why set; no machine at all is an
// error. The run keeps each request built from an input under its run ask (compositionRun.machine).
func (c *compositionInputs) gather(ctx context.Context, run meshwants.StepRun, away func(node string) bool) ([]nodeInput, error) {
key, err := candidateKeyOf(run)
if err != nil {
return nil, err
}
nodes := c.machines(run.Module, run.Branch)
if len(nodes) == 0 {
return nil, fmt.Errorf("no node places %s on the branch %q, so the composition step has no machine to compose", run.Module, run.Branch)
}
// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the
// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set.
func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput {
bound := c.bound
if bound <= 0 {
bound = nec.AnswerBound
}
out := make([]nodeInput, len(nodes))
var wg sync.WaitGroup
for i, node := range nodes {
if away != nil && away(node) {
out[i] = nodeInput{Node: node, Away: true}
continue
}
wg.Add(1)
go func() {
defer wg.Done()
out[i] = c.one(ctx, bound, node, key)
}()
}
wg.Wait()
return out, nil
return c.one(ctx, bound, node, candidate)
}
// one asks one node for its input.
@@ -68,7 +68,7 @@ func TestTheMachinesOfAStep(t *testing.T) {
// §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a
// silent or unready node says why, and a machine away is not asked.
func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
// gather asks the machines side by side, so the record of what each was asked is guarded.
// the machines are asked side by side, so the record of what each was asked is guarded.
var mu sync.Mutex
asked := map[string]string{}
call := func(key string, body any) (json.RawMessage, error) {
@@ -86,7 +86,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second}
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
if err != nil || len(got) != 4 {
t.Fatalf("%+v %v", got, err)
}
@@ -105,7 +105,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
t.Errorf("the machine away is asked or not said away: %+v", got[3])
}
none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call}
if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
if _, err := gatherAll(none, meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
t.Error("a step with no machine is gathered")
}
}
@@ -124,7 +124,7 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
return json.RawMessage(answer + `,"file":"` + file + `"}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second}
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
if err != nil || len(got) != 2 {
t.Fatalf("%+v %v", got, err)
}
@@ -135,3 +135,28 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
t.Errorf("g14: %+v", got[1])
}
}
// gatherAll asks each machine of a step for its input as each run's start does (compositionInputs.of), side by side; a machine away
// is not asked; no machine is an error.
func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bool) ([]nodeInput, error) {
key, err := candidateKeyOf(run)
if err != nil {
return nil, err
}
nodes := c.machines(run.Module, run.Branch)
if len(nodes) == 0 {
return nil, errors.New("no machine")
}
out := make([]nodeInput, len(nodes))
var wg sync.WaitGroup
for i, node := range nodes {
if away != nil && away(node) {
out[i] = nodeInput{Node: node, Away: true}
continue
}
wg.Add(1)
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }()
}
wg.Wait()
return out, nil
}
@@ -21,12 +21,14 @@ import (
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The run of a composition step (design 47 G §354, §63, §64; A-226, A-238, A-240, A-243, A-252, A-254, A-261): one
// composer.Request per machine, depth compose, the one machine and one case `change`, its manifests, verdicts and seats from the
// step's catalog revision read through mesh-catalog's `revision`; one run ask per machine of the seat node-build-go, naming no
// head, its program the node-engine build's catalog entry with the composer's package; the run's end heard on the build seat's
// step-ended of that ask, its answer (a composer.Verdict) read inline from `output`. A machine the watchman said away
// (machine-away) is not composed and passes with a warning (§64).
// The run of a composition step (design 47 G §354, §63, §64, §718; A-226, A-238, A-240, A-252, A-254, A-261): the step reads one
// composition run per machine of its candidate (compositionruns.go), and the step that starts a machine's run composes it: one
// compose-candidate ask of the node, one composer.Request, depth compose, the one machine and one case `change`, its manifests,
// verdicts and seats from the step's catalog revision read through mesh-catalog's `revision`, kept once in `inputs`; one run ask
// of the seat node-build-go, naming no head, its program the node-engine build's catalog entry with the composer's package,
// chosen once and recorded in the run. The run's end is heard on the build seat's step-ended of that ask, its answer (a
// composer.Verdict) read from `output`, and written in the run's key, whose change every reading step hears. A machine the
// watchman said away (machine-away) is not composed and passes with a warning (§64).
const (
// runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run
@@ -41,42 +43,6 @@ const (
seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions
)
// runEnds hands each build seat's step-ended of an ask to the composition run that waits for it (A-240), by the ask's id.
type runEnds struct {
mu sync.Mutex
waits map[string]chan buildstep.StepEnded
}
// expect is the end of the ask id, once heard; called before the ask is put, so no end is missed.
func (r *runEnds) expect(id string) <-chan buildstep.StepEnded {
r.mu.Lock()
defer r.mu.Unlock()
if r.waits == nil {
r.waits = map[string]chan buildstep.StepEnded{}
}
ch := make(chan buildstep.StepEnded, 1)
r.waits[id] = ch
return ch
}
// forget stops waiting for the ask id.
func (r *runEnds) forget(id string) {
r.mu.Lock()
delete(r.waits, id)
r.mu.Unlock()
}
// heard hands an end to the run that waits for its ask; one nobody waits for is another asker's, left.
func (r *runEnds) heard(e buildstep.StepEnded) {
r.mu.Lock()
ch := r.waits[e.Ask]
delete(r.waits, e.Ask)
r.mu.Unlock()
if ch != nil {
ch <- e
}
}
// awayNodes are the machines the watchman said away (machine-away of the seat mesh-watchdogs, §389) and not back since.
type awayNodes struct {
mu sync.Mutex
@@ -109,6 +75,8 @@ func (a *awayNodes) is(machine string) bool {
// compositionRun runs a composition step.
type compositionRun struct {
// runs are the composition runs of the state composition-runs (compositionruns.go).
runs *compositionRuns
inputs *compositionInputs
keeper *inputKeeper
// call asks a seat verb held once for the mesh (process.Conn.CallKept).
@@ -119,8 +87,7 @@ type compositionRun struct {
running func(node, module string) (ne.Running, bool)
away *awayNodes
// put puts a run ask on the build seat's check queue under the assigner's name.
put func(ctx context.Context, a buildstep.StepAsk) error
ends *runEnds
put func(ctx context.Context, a buildstep.StepAsk) error
// reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer).
reads *compositionReads
sleep func(ctx context.Context, d time.Duration) error
@@ -132,46 +99,81 @@ type compositionRun struct {
// machineEnd is what one machine came to.
type machineEnd struct {
failed, couldNot, warning string
failed, couldNot string
warnings []string
// judged is set where the composer judged the machine, composing or not (A-467a).
judged bool
}
// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine
// could not be composed or was not judged, else passed with a warning per machine away.
func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome {
half, err := r.revision(ctx, run)
if err != nil {
return couldNotRun("the step's catalog revision is not read: " + err.Error())
}
seats, err := r.seats(ctx, half.Revision.Definitions)
// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's
// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not
// compose, could not run when a machine could not be composed or was not judged, else passed with a warning per machine away.
func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
candidate, err := candidateKeyOf(step)
if err != nil {
return couldNotRun(err.Error())
}
manifests, verdicts, err := caseOf(half)
if err != nil {
return couldNotRun(err.Error())
nodes := r.inputs.machines(step.Module, step.Branch)
if len(nodes) == 0 {
return couldNotRun(fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch))
}
programOf, err := r.programs(ctx, half)
if err != nil {
return couldNotRun(err.Error())
}
inputs, err := r.inputs.gather(ctx, run, r.away.is)
if err != nil {
return couldNotRun(err.Error())
reader := readerOf(id, step)
prep := &runPrep{r: r, step: step}
var ends []machineEnd
var keys []string
for _, node := range nodes {
if r.away.is(node) {
ends = append(ends, machineEnd{warnings: []string{node + " is away (the watchman's machine-away), so it is not composed"}})
continue
}
join := r.runs.join
if step.Rerun { // a step asked again (§727)
join = r.runs.rejoin
}
j, err := join(reader, candidate, node, buildstep.NewID())
if err != nil {
ends = append(ends, machineEnd{couldNot: node + ": " + err.Error()})
continue
}
if j.started {
go r.start(ctx, j.key, j.run, prep)
}
keys = append(keys, j.key)
}
return r.read(ctx, step.Module, keys, ends)
}
// read waits for the end of each run of keys and judges the step of module from them, naming in its progress and its end the
// runs it reads and their asks (§723, §729).
func (r *compositionRun) read(ctx context.Context, module string, keys []string, ends []machineEnd) stepOutcome {
setShared(ctx, r.runs.sharedOf(keys, nil))
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
ends := make([]machineEnd, len(inputs))
var wg sync.WaitGroup
for i, in := range inputs {
wg.Add(1)
go func() {
defer wg.Done()
ends[i] = r.machine(ctx, in, buildstep.NewID(), programOf, manifests, verdicts, seats)
}()
}
wg.Wait()
runs := r.runs.await(ctx, keys)
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
for _, key := range keys {
ends = append(ends, judgeRun(module, runs[key]))
}
out := outcomeOf(ends)
out.Shared = r.runs.sharedOf(keys, runs)
return out
}
// resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended.
func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
candidate, err := candidateKeyOf(step)
if err != nil {
return couldNotRun(err.Error())
}
keys, err := r.runs.keysOf(candidate, id)
if err != nil {
return couldNotRun("the step's composition runs are not read: " + err.Error())
}
return r.read(ctx, step.Module, keys, nil)
}
// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no
// machine never passes (A-467a), each machine not judged named with why.
func outcomeOf(ends []machineEnd) stepOutcome {
var out stepOutcome
var failed, couldNot []string
judged := 0
@@ -181,14 +183,14 @@ func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRu
}
failed = appendSome(failed, e.failed)
couldNot = appendSome(couldNot, e.couldNot)
out.Warnings = appendSome(out.Warnings, e.warning)
out.Warnings = append(out.Warnings, e.warnings...)
}
switch {
case len(failed) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ")
case len(couldNot) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ")
case judged == 0: // a step that judged no machine never passes (A-467a): each machine not judged is named with why
case judged == 0:
said := "no machine was judged"
if len(out.Warnings) == 0 {
said += ": the step has no machine"
@@ -213,82 +215,187 @@ func couldNotRun(said string) stepOutcome {
return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said}
}
// machine composes one machine: its request kept under its ask, the ask put, its end heard and its verdict read.
func (r *compositionRun) machine(ctx context.Context, in nodeInput, ask string, programOf func(node string) (buildstep.RunProgram, error),
manifests map[string]composer.Manifest, verdicts map[string]json.RawMessage, seats []json.RawMessage) machineEnd {
node := in.Node
switch {
case in.Away:
return machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"}
case in.Why != "":
return machineEnd{couldNot: node + " gave no composition input: " + in.Why}
// runPrep is what the step that starts a run reads of its catalog revision, once for every machine it starts: the seats, the
// case's manifests and verdicts, and the composer program per machine.
type runPrep struct {
r *compositionRun
step meshwants.StepRun
once sync.Once
// read once
seats []json.RawMessage
manifests map[string]composer.Manifest
verdicts map[string]json.RawMessage
none map[string]string
programOf func(node string) (buildstep.RunProgram, string, error)
err error
}
func (p *runPrep) get(ctx context.Context) error {
p.once.Do(func() {
half, err := p.r.revision(ctx, p.step)
if err != nil {
p.err = errors.New("the step's catalog revision is not read: " + err.Error())
return
}
if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil {
return
}
if p.manifests, p.verdicts, p.none, p.err = caseOf(half); p.err != nil {
return
}
p.programOf, p.err = p.r.programs(ctx, half)
})
return p.err
}
// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`,
// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run.
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) {
node, ask := run.Machine, run.Ask
end := func(said string) {
if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil {
r.logf("the composition run %s is not ended: %v", key, err)
}
}
program, err := programOf(node)
if err := prep.get(ctx); err != nil {
end(node + ": " + err.Error())
return
}
in := r.inputs.of(ctx, node, run.Candidate)
if in.Why != "" {
end(node + " gave no composition input: " + in.Why)
return
}
program, engineFp, err := prep.programOf(node)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
end(node + ": " + err.Error())
return
}
req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node},
Cases: []composer.Case{{Name: changeCase, Manifests: manifests, Verdicts: verdicts}}, Seats: seats}
Cases: []composer.Case{{Name: changeCase, Manifests: prep.manifests, Verdicts: prep.verdicts, None: prep.none}}, Seats: prep.seats,
BlameTries: r.runs.set.blameTries} // the composer's bound of its search for culprits (§721, A-466r)
raw, err := json.Marshal(req)
if err != nil {
return machineEnd{couldNot: node + ": its request cannot be encoded: " + err.Error()}
end(node + ": its request cannot be encoded: " + err.Error())
return
}
if _, err := r.keeper.keep(ask, raw); err != nil {
digest, err := r.keeper.keep(ask, raw)
if err != nil {
r.logf("the composition request of %s is refused: %v", node, err)
return machineEnd{couldNot: node + ": its request is not kept: " + err.Error()}
end(node + ": its request is not kept: " + err.Error())
return
}
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
end(node + ": " + err.Error())
return
}
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: engineFp}
c.Input = digest
return true
}); err != nil {
end(node + ": the run's program and input are not written: " + err.Error())
return
}
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit}}
ended := r.ends.expect(ask)
defer r.ends.forget(ask)
if err := r.put(ctx, a); err != nil {
return machineEnd{couldNot: node + ": its run ask " + ask + " is not put: " + err.Error()}
end(node + ": its run ask " + ask + " is not put: " + err.Error())
}
var e buildstep.StepEnded
select {
case <-ctx.Done():
return machineEnd{couldNot: node + ": the step stopped before its run ask " + ask + " ended"}
case e = <-ended:
}
if e.Outcome != buildstep.OutcomePassed {
return machineEnd{couldNot: fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, ask, e.Outcome, e.Node, e.Said)}
}
answer, err := e.ReadOutput(buildstep.AskGo, r.ask) // the program's last line of standard output (A-254)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
}
return verdictOf(node, answer)
}
// verdictOf reads the composer's verdict of one machine: not composing is failed, naming why; not judged is could not run, naming why (A-467).
func verdictOf(node string, answer []byte) machineEnd {
// notTried is the problem the composer names a module with that its search for culprits did not try, past the request's
// blame_tries (design 47 G §721; the composer's composer.NotTried in mesh-host, compared by its words, A-470b).
const notTried = "not tried: the search reached its bound"
// judgeRun judges one machine of the step of module from the run it read (§721, §722): could not run with the run's words; else
// from the machine's composer verdict: not judged is could not run (A-467); a verdict blaming the step's module fails it, or is
// could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on
// it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the
// composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there.
func judgeRun(module string, run meshwants.CompositionRun) machineEnd {
node := run.Machine
if run.Ended == nil {
return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"}
}
if run.Outcome != meshwants.CompositionRunOutcomePassed {
return machineEnd{couldNot: run.Said}
}
m, err := machineOf(run.Verdict)
if err != nil {
return machineEnd{couldNot: node + ": the run's verdict does not read: " + err.Error()}
}
if m.NotJudged != "" { // a step never passes on a machine it did not judge (A-467)
return machineEnd{couldNot: node + " was not judged: " + m.NotJudged}
}
why := strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")
switch {
case m.Blamed == nil && !m.Composes: // an old composer (§721)
return machineEnd{judged: true, failed: node + " does not compose: " + why, warnings: []string{node + ": the composer names no culprits"}}
case len(m.Blamed) == 0 && !m.Composes: // the node's own account (A-452)
return machineEnd{judged: true, failed: node + " does not compose and blames no module: " + why}
}
end := machineEnd{judged: true}
var own, others []string
for _, b := range m.Blamed {
switch {
case b.Module == module && b.Problem == notTried:
end.couldNot = fmt.Sprintf("%s holds %s untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", node, module)
case b.Module == module:
own = append(own, b.Problem)
default:
others = append(others, node+" holds "+b.Module+": "+b.Problem)
}
}
if len(own) > 0 {
end.failed, end.couldNot = node+" holds "+module+": "+strings.Join(own, "; "), ""
}
end.warnings = others
return end
}
// machineOf is the composer's verdict of one machine as a run keeps it.
func machineOf(v map[string]any) (composer.Machine, error) {
var m composer.Machine
raw, err := json.Marshal(v)
if err == nil {
err = json.Unmarshal(raw, &m)
}
return m, err
}
// runVerdict reads the composer's answer of one machine: its verdict as the run keeps it, or why it is none.
func runVerdict(node string, answer []byte) (map[string]any, string) {
var v composer.Verdict
if err := json.Unmarshal(answer, &v); err != nil {
return machineEnd{couldNot: node + ": the composer answered no verdict: " + err.Error()}
return nil, node + ": the composer answered no verdict: " + err.Error()
}
if v.V != composer.Version {
return machineEnd{couldNot: fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)}
return nil, fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)
}
for _, c := range v.Cases {
if c.Name != changeCase {
continue
}
m, ok := c.Machines[node]
switch {
case !ok:
return machineEnd{couldNot: node + ": the composer's verdict names no such machine"}
case m.NotJudged != "": // a step never passes on a machine it did not judge (A-467)
return machineEnd{couldNot: node + " was not judged: " + m.NotJudged}
case !m.Composes:
return machineEnd{judged: true, failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")}
if !ok {
return nil, node + ": the composer's verdict names no such machine"
}
return machineEnd{judged: true}
raw, err := json.Marshal(m)
var out map[string]any
if err == nil {
err = json.Unmarshal(raw, &out)
}
if err != nil {
return nil, node + ": the composer's verdict cannot be kept: " + err.Error()
}
return out, ""
}
return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase}
return nil, node + ": the composer's verdict has no case " + changeCase
}
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of
@@ -400,20 +507,28 @@ func (r *compositionRun) seatAt(ctx context.Context, seat, digest string) (json.
return def, nil
}
// caseOf is the case's manifests and verdicts from the revision's entries: each manifest whole, and the verdict of each one not
// accepted (a module without one is accepted).
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, error) {
// caseOf is the case's manifests, verdicts and none from the revision's entries: each manifest whole, the verdict of each one not
// accepted (a module without one is accepted), and the revision's none words of each module it gives no fingerprint, whose
// build can never be made (design 47 G A-470j, A-470k).
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, map[string]string, error) {
manifests := map[string]composer.Manifest{}
verdicts := map[string]json.RawMessage{}
var none map[string]string
for name, e := range half.Entries {
if len(e.Manifest) == 0 {
continue
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
return nil, nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
}
manifests[name] = composer.Manifest{Raw: raw}
if e.Fp == "" && e.None != "" {
if none == nil {
none = map[string]string{}
}
none[name] = e.None
}
if e.Verdict == "" || e.Verdict == mcc.VerdictAccepted {
continue
}
@@ -426,11 +541,11 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMes
}
raw, err = json.Marshal(v)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
verdicts[name] = raw
}
return manifests, verdicts, nil
return manifests, verdicts, none, nil
}
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
@@ -468,50 +583,53 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil
}
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451).
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
// programs answers, per machine, the composer its run builds and the node-engine fingerprint it was chosen from (A-226, A-243,
// A-252, §719): where the revision's node-engine entry differs from the trunk's, that build for every machine; else the build the
// machine's node-engine runs, as its `running` says, its entry read through mesh-catalog's `entry`; a machine whose node-engine's
// `running` is not heard gets the revision's entry, said (A-451).
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, error), error) {
engine, ok := nodeEngineEntry(half.Entries)
if !ok {
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
}
if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() {
p, err := programOfEntry(engine)
return func(string) (buildstep.RunProgram, error) { return p, err }, nil
return func(string) (buildstep.RunProgram, string, error) { return p, engine.Fp, err }, nil
}
var mu sync.Mutex
byFp := map[string]buildstep.RunProgram{}
return func(node string) (buildstep.RunProgram, error) {
return func(node string) (buildstep.RunProgram, string, error) {
running, ok := r.running(node, engine.Module)
if !ok || running.Fingerprint == "" {
r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node,
engine.Module, engine.Fp)
return programOfEntry(engine)
p, err := programOfEntry(engine)
return p, engine.Fp, err
}
fp := running.Fingerprint
mu.Lock()
p, ok := byFp[running.Fingerprint]
p, ok := byFp[fp]
mu.Unlock()
if ok {
return p, nil
return p, fp, nil
}
what := "the entry of the node-engine's build " + running.Fingerprint + " is not read"
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
what := "the entry of the node-engine's build " + fp + " is not read"
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: fp})
if err != nil || body == nil {
return buildstep.RunProgram{}, unreadErr(what, err, said)
return buildstep.RunProgram{}, fp, unreadErr(what, err, said)
}
var a meshcatalog.EntryAnswer
if err := json.Unmarshal(body, &a); err != nil {
return buildstep.RunProgram{}, fmt.Errorf("%s: %w", what, err)
return buildstep.RunProgram{}, fp, fmt.Errorf("%s: %w", what, err)
}
p, err = programOfEntry(a.Entry)
if err != nil {
return p, err
return p, fp, err
}
mu.Lock()
byFp[running.Fingerprint] = p
byFp[fp] = p
mu.Unlock()
return p, nil
return p, fp, nil
}, nil
}
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"sync"
"testing"
@@ -30,12 +31,16 @@ type runStand struct {
requests map[string]composer.Request // by node
entries []string // the fingerprints `entry` was asked
r *compositionRun
runs *compositionRuns
puts int // the run asks put
// noDefinitions: the revision names no seat definition (A-234's stand-in); trunkSeats: the trunk's seat list as (seat, digest)
// pairs; unjudged: the machine the composer does not judge; listFault: the trunk's seat list cannot be read.
noDefinitions bool
trunkSeats []meshcatalog.SeatDefinitionsAnswerSeatsItem
unjudged string
listFault error
// verdict, when set, is the composer's verdict of a machine (by node), else the stand's own.
verdict func(node string) (composer.Machine, bool)
}
func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand {
@@ -52,7 +57,8 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
"": {Revision: &meshcatalog.CatalogRevision{Revision: 7, Branch: "main", Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "d1"}}},
Entries: []meshcatalog.CatalogEntry{engine(revisionEngine, revisionCommit), {Module: "bad", Verdict: mcc.VerdictRefused, Inputs: &meshcatalog.EntryInputs{Problems: []string{"no seat x"}},
Manifest: map[string]any{"module": "bad"}}}, Next: "bad"},
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}}}},
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}},
{Module: "lidarr", None: "builders disagree: node", Manifest: map[string]any{"module": "lidarr"}}}}, // no fingerprint (A-470j)
}
call := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
@@ -99,10 +105,10 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
away := &awayNodes{}
away.set("shanks", true)
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
ends := &runEnds{}
s.r = &compositionRun{
s.runs = newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
s.r = &compositionRun{runs: s.runs,
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
keeper: keeper, reads: &compositionReads{}, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
keeper: keeper, reads: &compositionReads{}, call: call, away: away, now: time.Now, logf: t.Logf,
trunk: func() (mcc.Half, bool) {
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
},
@@ -124,13 +130,27 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
s.asks[node], s.requests[node] = a, req
s.mu.Unlock()
m := composer.Machine{Composes: true}
if node == s.unjudged {
override, set := composer.Machine{}, false
if s.verdict != nil {
override, set = s.verdict(node)
}
if set {
m = override
} else if node == s.unjudged {
m = composer.Machine{NotJudged: "the composer was handed no seat list"}
} else if node == "g14" {
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}}
}
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
go ends.heard(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)})
s.mu.Lock()
s.puts++
s.mu.Unlock()
go func() {
if err := s.runs.heardEnd(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)},
func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
t.Error(err)
}
}()
return nil
},
}
@@ -146,7 +166,8 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
out := s.r.run(context.Background(), "ask-step", step)
if out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") || !strings.Contains(out.Said, "/etc/x") ||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 1 || !strings.Contains(out.Warnings[0], "shanks is away") {
strings.Contains(out.Said, "ace") || len(out.Warnings) != 2 || !strings.Contains(out.Warnings[0], "shanks is away") ||
out.Warnings[1] != "g14: the composer names no culprits" {
t.Fatalf("the step came to %+v", out)
}
if len(s.asks) != 2 {
@@ -160,12 +181,12 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
if s.asks["g14"].Run.Program.Commit != commitE2 {
t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit)
}
if req.Depth != composer.DepthCompose || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
if req.Depth != composer.DepthCompose || req.BlameTries != 20 || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` {
t.Errorf("the request of ace %+v", req)
}
c := req.Cases[0]
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 3 || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) ||
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 4 || len(c.None) != 1 || c.None["lidarr"] != "builders disagree: node" || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) ||
!strings.Contains(string(c.Verdicts["bad"]), "no seat x") {
t.Errorf("the case %+v %s", c.Manifests, c.Verdicts)
}
@@ -329,3 +350,227 @@ func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T)
})
}
}
// Design 47 G §718, §719, §720: three composition steps of one revision on two machines start two runs, one request and one run
// ask per machine, and each step reads both verdicts; the run records its program with the node-engine fingerprint it was chosen
// from and the digest of its kept request, which names no module of the asking pipeline; a step of another candidate starts its
// own runs, and a run that could not run is never read again.
func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
var wg sync.WaitGroup
outs := make([]stepOutcome, 3)
for i, m := range []string{"m", "m2", "m3"} {
wg.Add(1)
go func() {
defer wg.Done()
outs[i] = s.r.run(context.Background(), "ask-"+m, meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
}()
}
wg.Wait()
for i, o := range outs {
if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" ||
o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask
t.Errorf("step %d names the runs %+v", i, o.Shared)
}
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") {
t.Errorf("step %d came to %+v, want both machines read", i, o)
}
}
if s.puts != 2 {
t.Fatalf("%d run asks put, want one per machine", s.puts)
}
ace, _, _, _ := s.runs.read(runKeyOf("main.7", "ace"))
g14, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
if len(ace.Readers) != 3 || len(g14.Readers) != 3 || ace.Program == nil || ace.Program.Fingerprint != "src2:e1" || ace.Program.Commit != commitE1 ||
g14.Program == nil || g14.Program.Fingerprint != "src2:e2" || ace.Input == "" || ace.Input != digestOf(mustJSON(t, s.requests["ace"])) ||
ace.Outcome != meshwants.CompositionRunOutcomePassed || g14.Outcome != meshwants.CompositionRunOutcomePassed || g14.Verdict["composes"] != false {
t.Errorf("the runs: ace %+v, g14 %+v", ace, g14)
}
if req := s.requests["ace"]; strings.Contains(string(mustJSON(t, req)), "main.7.m") || string(req.Input) != `{"node":"ace"}` {
t.Errorf("the request names the asking pipeline or not the node's input: %s", mustJSON(t, req))
}
// Another candidate (a run on request) starts its own runs.
if o := s.r.run(context.Background(), "ask-r", meshwants.StepRun{Pipeline: "run-r1.m", Module: "m", Branch: "main", Revision: 7, Run: "r1", Step: meshwants.StepRunStepComposition}); o.Outcome != meshwants.StepEndedOutcomeFailed || s.puts != 4 {
t.Errorf("a step of another candidate came to %+v with %d asks, want its own two runs", o, s.puts)
}
// A run whose ask could not be put could not run; the next step of the candidate starts it again.
f := newRunStand(t, "src2:e1", "src2:e1", "d1")
f.r.inputs.machines = func(string, string) []string { return []string{"ace"} }
put, refused := f.r.put, 0
f.r.put = func(ctx context.Context, a buildstep.StepAsk) error {
if refused == 0 {
refused++
return errors.New("no stream")
}
return put(ctx, a)
}
first := f.r.run(context.Background(), "ask-1", meshwants.StepRun{Pipeline: "main.7.m", Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
second := f.r.run(context.Background(), "ask-2", meshwants.StepRun{Pipeline: "main.7.m2", Module: "m2", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
if first.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(first.Said, "is not put") || second.Outcome != meshwants.StepEndedOutcomePassed || f.puts != 1 {
t.Errorf("after a run that could not run: %+v then %+v, %d asks", first, second, f.puts)
}
}
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return raw
}
// Design 47 G §721, §722, A-470b: a step judges each machine from its verdict's `blamed`: a verdict blaming the step's module fails
// it and passes another module's step with a warning naming the module; a machine that does not compose and blames no module fails
// every step; a verdict with no `blamed` is judged as before and says "the composer names no culprits"; a module the search did not
// try is could not run for its own step; a run that could not run is could not run.
func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) {
at := time.Now()
runOf := func(verdict string) meshwants.CompositionRun {
var v map[string]any
if err := json.Unmarshal([]byte(verdict), &v); err != nil {
t.Fatal(err)
}
return meshwants.CompositionRun{Candidate: "main.7", Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomePassed, Verdict: v}
}
blamesA := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and c's"}]}`
for _, tc := range []struct {
name, module string
run meshwants.CompositionRun
failed, couldNo string
warnings []string
}{
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil},
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}},
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil},
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}},
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil},
{"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "",
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil},
{"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "",
[]string{"ace holds a: not tried: the search reached its bound"}},
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil},
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil},
} {
t.Run(tc.name, func(t *testing.T) {
e := judgeRun(tc.module, tc.run)
if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) {
t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings)
}
})
}
}
// Design 47 G §727, A-468g, A-470d: a rerun of a composition step starts a new run only on the machine it did not pass, replacing
// that key and carrying every former reader, the one that did not pass marked rerun with the rerun's caller; at the new run's end
// that reader says its end again, marked rerun, naming the new run's ask; a reader that passed does not move; the machine the
// step passed keeps its verdict.
func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
var mu sync.Mutex
fixed := false
s.verdict = func(node string) (composer.Machine, bool) {
mu.Lock()
defer mu.Unlock()
m := composer.Machine{Composes: true}
m.Blamed = []struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{}
if node == "g14" && !fixed {
m.Composes, m.Reason = false, "two modules own one path"
m.Blamed = append(m.Blamed, struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{"a", "/etc/x"}, struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{"d", "/etc/x"})
}
return m, true
}
ends := make(chan meshwants.StepEnded, 16)
msgIDs := map[string]string{} // the message id of each step's latest end
steps := &compositionSteps{node: "n1", now: time.Now, logf: t.Logf, run: s.r.run, resume: s.r.resume, ended: s.runs.stepEnded,
say: func(event string, body any, msgID string) error {
if e, ok := body.(meshwants.StepEnded); ok {
mu.Lock()
msgIDs[e.ID] = msgID
mu.Unlock()
ends <- e
}
return nil
}}
move := steps.move(context.Background())
s.runs.moved.Store(&move)
step := func(m string, rerun bool) []byte {
raw, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition, Rerun: rerun, By: map[bool]string{true: "op@n1"}[rerun]})
return raw
}
got := map[string]meshwants.StepEnded{}
for _, m := range []string{"a", "b", "d"} {
steps.take(context.Background(), "ask-"+m, step(m, false))
e := <-ends
got[e.Module] = e
}
if got["a"].Outcome != meshwants.StepEndedOutcomeFailed || got["d"].Outcome != meshwants.StepEndedOutcomeFailed || got["b"].Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("the first ends: %+v", got)
}
former := got["d"].Shared[1].Ask
mu.Lock()
fixed = true
mu.Unlock()
steps.take(context.Background(), "ask-a2", step("a", true))
moved := map[string]meshwants.StepEnded{}
for range 2 {
select {
case e := <-ends:
moved[e.ID] = e
case <-time.After(5 * time.Second):
t.Fatalf("the ends after the rerun: %+v", moved)
}
}
a, d := moved["ask-a2"], moved["ask-d"]
if a.Outcome != meshwants.StepEndedOutcomePassed || d.Outcome != meshwants.StepEndedOutcomePassed || !d.Rerun || d.By != "op@n1" ||
len(d.Shared) != 2 || d.Shared[1].Machine != "g14" || d.Shared[1].Ask == former || d.Shared[1].Ask != a.Shared[1].Ask {
t.Fatalf("after the rerun: a %+v, d %+v", a, d)
}
mu.Lock()
// A-470q: the first end keeps the ask's id; the end said again names the moved run's ask, so the bus does not drop it.
if msgIDs["ask-a2"] != "ask-a2" || msgIDs["ask-d"] != "ask-d."+a.Shared[1].Ask || msgIDs["ask-b"] != "ask-b" {
t.Errorf("the message ids of the ends: %v", msgIDs)
}
mu.Unlock()
if s.puts != 3 {
t.Errorf("%d run asks, want one more for the machine a did not pass", s.puts)
}
select {
case e := <-ends:
t.Errorf("a reader that passed moved: %+v", e)
case <-time.After(50 * time.Millisecond):
}
run, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
marks := map[string]bool{}
for _, c := range run.Readers {
marks[c.ID] = c.Rerun
}
if len(run.Readers) != 3 || !marks["ask-d"] || marks["ask-b"] || !marks["ask-a2"] {
t.Errorf("the new run's readers %+v", run.Readers)
}
// A-470m: each reader's `ended` is written once its step said its end, and ask-b's carried into the new run.
deadline := time.After(5 * time.Second)
for {
wake := s.runs.wake()
run, _, _, _ = s.runs.read(runKeyOf("main.7", "g14"))
if !slices.ContainsFunc(run.Readers, func(c meshwants.CompositionReader) bool { return c.Ended == nil }) {
break
}
select {
case <-wake:
case <-deadline:
t.Fatalf("a reader of the new run names no end: %+v", run.Readers)
}
}
}
@@ -0,0 +1,896 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/buckets"
)
// The composition runs (design 47 G §718, §719; A-468, A-469d, A-469w, A-469x): the assigner composes each machine once per
// candidate, whatever number of composition steps name it. Each run is one key of its old state `composition-runs`,
// `<candidate key>.<machine>`, each part escaped as a key's name; the step that starts a run writes the key create-only,
// so two steps of one candidate starting at once start one run and the second reads it. A reader is the composition step's
// StepRun with its ask's id (A-468), added and removed by compare-and-set. A key is deleted once composition-runs-kept has
// passed since its `ended`, judged at each new run over at most 20 of the oldest ended keys, as the state inputs is
// (A-465b, A-468b); at the cap (composition-runs-cap-mib, the manifest's max-mib its ceiling, A-469d) no run is started.
const (
// runsSwept is how many of the oldest ended keys each new run judges.
runsSwept = 20
// runBoundPrefix and runUntakenPrefix open a run's two bound keys, each with its own time to live (§720, A-468b,
// A-470): `bound.<run key>` from the ask's ask-taken, `untaken.<run key>` from the run's start.
runBoundPrefix = "bound."
runUntakenPrefix = "untaken."
)
// runKeyOf is the key of a candidate's composition run of a machine (§719).
func runKeyOf(candidate, machine string) string {
return keysPart(candidate) + "." + keys.EscapeName(machine)
}
// keysPart is a name as a key's part.
func keysPart(name string) string { return keys.EscapeName(name) }
// isRunKey is whether a key of the state is a run's own, not one of its bound keys.
func isRunKey(key string) bool {
return !strings.HasPrefix(key, runBoundPrefix) && !strings.HasPrefix(key, runUntakenPrefix)
}
// runSettings are the settings of the composition runs, each read at the start from the file the manifest declares.
type runSettings struct {
// capBytes is composition-runs-cap-mib in bytes.
capBytes int64
// kept is composition-runs-kept, stepBound composition-step-bound, untaken run-untaken-bound (A-469x).
kept, stepBound, untaken time.Duration
// blameTries is blame-tries, carried in each run's request (§721).
blameTries int
}
// defaultRunSettings are the settings' defaults, as the manifest gives them.
var defaultRunSettings = runSettings{capBytes: 16 << 20, kept: 24 * time.Hour, stepBound: 30 * time.Minute, untaken: 24 * time.Hour, blameTries: 20}
// runSettingsOf reads the settings composition-runs-cap-mib, composition-runs-kept, composition-step-bound,
// run-untaken-bound and blame-tries from the files the environment names (A-469x, as outsideSettings): a missing or empty
// file is its default; a content that is no positive whole number or Go duration is an error naming the file.
func runSettingsOf(getenv func(string) string) (runSettings, error) {
s := defaultRunSettings
mib, err := settingFile(getenv("MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE"))
if err != nil {
return s, err
}
if mib > 0 {
s.capBytes = int64(mib) << 20
}
tries, err := settingFile(getenv("MESH_ASSIGNER_BLAME_TRIES_FILE"))
if err != nil {
return s, err
}
if tries > 0 {
s.blameTries = tries
}
for env, to := range map[string]*time.Duration{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": &s.kept,
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": &s.stepBound, "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": &s.untaken} {
d, err := durationFile(getenv(env))
if err != nil {
return s, err
}
if d > 0 {
*to = d
}
}
return s, nil
}
// durationFile reads one setting's file holding a Go duration: 0 for no path, no file or an empty one.
func durationFile(path string) (time.Duration, error) {
if path == "" {
return 0, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return 0, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return 0, nil
}
d, err := time.ParseDuration(v)
if err != nil || d <= 0 {
return 0, fmt.Errorf("%s holds %q, which is not a positive Go duration such as 30m", path, v)
}
return d, nil
}
// errRunChanged is a compare-and-set of a run's key that met another write.
var errRunChanged = errors.New("the composition run's key changed")
// runsKV is the state composition-runs: the bus bucket, or a test's.
type runsKV interface {
// get is a key's value and revision; nil for a key not there.
get(key string) ([]byte, uint64, error)
// create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so.
create(key string, value []byte) (uint64, error)
update(key string, value []byte, revision uint64) (uint64, error)
// delAt deletes a key at its revision: errRunChanged when it no longer stands so, nothing for a key not there. No write
// of the state is made from a value read before its latest change.
delAt(key string, revision uint64) error
keys() ([]string, error)
// arm writes a bound key afresh holding the ask of the run it bounds, with its own time to live (§720, A-470n): at the key's
// latest revision, a delete marker counted, read again when another write came between; its expiry is the bus's event. It
// answers the revision written.
arm(key, ask string, ttl time.Duration) (uint64, error)
// used is how many bytes the state holds.
used() (int64, error)
}
// busRuns is runsKV over the bus bucket.
type busRuns struct {
ctx context.Context
js jetstream.JetStream
kv jetstream.KeyValue
}
func (b busRuns) get(key string) ([]byte, uint64, error) {
e, err := b.kv.Get(b.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, nil
}
if err != nil {
return nil, 0, err
}
return e.Value(), e.Revision(), nil
}
func (b busRuns) create(key string, value []byte) (uint64, error) {
rev, err := b.kv.Create(b.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return 0, errRunChanged
}
return rev, err
}
func (b busRuns) update(key string, value []byte, revision uint64) (uint64, error) {
rev, err := b.kv.Update(b.ctx, key, value, revision)
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return 0, errRunChanged
}
return rev, err
}
func (b busRuns) delAt(key string, revision uint64) error {
err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision))
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return errRunChanged
}
return err
}
func (b busRuns) keys() ([]string, error) {
l, err := b.kv.ListKeys(b.ctx)
if err != nil {
return nil, err
}
var out []string
for k := range l.Keys() {
out = append(out, k)
}
return out, nil
}
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
type boundValue struct {
Bound string `json:"bound"`
Ask string `json:"ask"`
}
// arm publishes the bound key holding ask with ttl as its own time to live, expecting the key's latest message (a delete
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
// takes its time to live with it, so the expiry is heard only ttl after the last write.
func (b busRuns) arm(key, ask string, ttl time.Duration) (uint64, error) {
if b.js == nil {
return 0, errors.New("no bus to arm a bound on")
}
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
if err != nil {
return 0, err
}
subject := "$KV." + b.kv.Bucket() + "." + key
for {
st, err := b.js.Stream(b.ctx, "KV_"+b.kv.Bucket())
if err != nil {
return 0, err
}
var last uint64
m, err := st.GetLastMsgForSubject(b.ctx, subject)
switch {
case err == nil:
last = m.Sequence
case !errors.Is(err, jetstream.ErrMsgNotFound):
return 0, err
}
ack, err := b.js.Publish(b.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
continue
}
if err != nil {
return 0, err
}
return ack.Sequence, nil
}
}
func (b busRuns) used() (int64, error) {
st, err := b.kv.Status(b.ctx)
if err != nil {
return 0, err
}
return int64(st.Bytes()), nil
}
// compositionRuns keeps the composition runs in the state composition-runs.
type compositionRuns struct {
kv runsKV
set runSettings
now func() time.Time
logf func(string, ...any)
mu sync.Mutex
// changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again.
changed chan struct{}
// seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is
// being written at that moment, so every lookup takes these beside it (keys).
seen map[string]bool
// say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119);
// nil says or drops nothing.
say publisher
drop func(ask, why string) error
// moved is told each run that ended, for the readers a rerun carried to say their end again (compositionSteps.move).
// Set once the steps are served, after the hub starts.
moved atomic.Pointer[func(run meshwants.CompositionRun)]
}
func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns {
return &compositionRuns{kv: kv, set: set, now: now, logf: logf, changed: make(chan struct{})}
}
// heard takes a change of the state from its watch (a module watches its own state, principle 1).
func (r *compositionRuns) heard() {
r.mu.Lock()
close(r.changed)
r.changed = make(chan struct{})
r.mu.Unlock()
}
// saw keeps a key the state's watch handed: standing, or deleted or expired.
func (r *compositionRuns) saw(key string, standing bool) {
r.mu.Lock()
defer r.mu.Unlock()
if r.seen == nil {
r.seen = map[string]bool{}
}
if standing {
r.seen[key] = true
} else {
delete(r.seen, key)
}
}
// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later
// is skipped by its reader.
func (r *compositionRuns) keys() ([]string, error) {
all, err := r.kv.keys()
if err != nil {
return nil, err
}
r.mu.Lock()
for key := range r.seen {
all = append(all, key)
}
r.mu.Unlock()
slices.Sort(all)
return slices.Compact(all), nil
}
// wake is closed at the next change heard.
func (r *compositionRuns) wake() <-chan struct{} {
r.mu.Lock()
defer r.mu.Unlock()
return r.changed
}
// read is a run's value and revision, or false.
func (r *compositionRuns) read(key string) (meshwants.CompositionRun, uint64, bool, error) {
raw, rev, err := r.kv.get(key)
if err != nil || raw == nil {
return meshwants.CompositionRun{}, 0, false, err
}
var run meshwants.CompositionRun
if err := json.Unmarshal(raw, &run); err != nil {
return meshwants.CompositionRun{}, 0, false, fmt.Errorf("the composition run %s does not read: %w", key, err)
}
return run, rev, true, nil
}
// change writes a run's key by compare-and-set: f changes the value read and says whether to write it; false leaves it.
// It answers the value written or left, and false for a key not there.
func (r *compositionRuns) change(key string, f func(*meshwants.CompositionRun) bool) (meshwants.CompositionRun, bool, error) {
for {
run, rev, ok, err := r.read(key)
if err != nil || !ok {
return run, ok, err
}
if !f(&run) {
return run, true, nil
}
raw, err := json.Marshal(run)
if err != nil {
return run, true, err
}
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return run, true, err
}
r.heard()
return run, true, nil
}
}
// fill is the state's fill against its cap.
func (r *compositionRuns) fill() (meshwants.Fill, error) {
at := r.now().UTC()
used, err := r.kv.used()
return meshwants.Fill{Store: buckets.CompositionRunsState, Bytes: used, Cap: r.set.capBytes, At: &at}, err
}
// joined is a reader's place in a run: the run's key, whether this reader started it (so it composes the machine), and the
// value as it stood.
type joined struct {
key string
started bool
run meshwants.CompositionRun
}
// join finds or starts the candidate's run of the machine for reader (§718, §719, §720): a run that stands and has not
// ended, or that ended other than could not run, is read, the reader added; none, or one that could not run, is started:
// written create-only (or replacing the ended key by compare-and-set), the reader its first, ask its run ask's id. At the
// cap no run is started, an error naming the fill.
func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
key := runKeyOf(candidate, machine)
for {
run, rev, ok, err := r.read(key)
if err != nil {
return joined{}, err
}
if ok && (run.Ended == nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) {
if hasReader(run.Readers, reader.ID) {
return joined{key: key, run: run}, nil
}
run.Readers = append(run.Readers, reader)
raw, err := json.Marshal(run)
if err != nil {
return joined{}, err
}
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
r.heard()
if run.Ended == nil && r.untaken(key, run.Ask) {
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
}
return joined{key: key, run: run}, nil
}
if err := r.room(); err != nil {
return joined{}, err
}
started := r.now().UTC()
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started,
Readers: []meshwants.CompositionReader{reader}}
raw, err := json.Marshal(fresh)
if err != nil {
return joined{}, err
}
if ok {
_, err = r.kv.update(key, raw, rev) // could not run is never read: a new run replaces it (§720)
} else {
_, err = r.kv.create(key, raw)
}
if errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
r.heard()
// An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470, A-470n).
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
}
r.sayWaits(fresh, fresh.Readers, "")
return joined{key: key, started: true, run: fresh}, nil
}
}
// room sweeps the state and says whether a new run may start: an error naming the fill at the cap (as A-465a).
func (r *compositionRuns) room() error {
if err := r.sweep(); err != nil {
r.logf("composition-runs: the keys past composition-runs-kept are not all deleted: %v", err)
}
f, err := r.fill()
if err != nil {
return fmt.Errorf("the fill of the state %s is not read, so no composition run is started: %w", buckets.CompositionRunsState, err)
}
if f.Bytes >= f.Cap {
return fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so no composition run is started", buckets.CompositionRunsState,
f.Cap>>20, f.Bytes)
}
return nil
}
// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys.
func (r *compositionRuns) sweep() error {
all, err := r.keys()
if err != nil {
return err
}
type endedAt struct {
key string
at time.Time
}
var ended []endedAt
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err != nil || !ok || run.Ended == nil {
continue
}
ended = append(ended, endedAt{key, *run.Ended})
}
slices.SortFunc(ended, func(a, b endedAt) int { return a.at.Compare(b.at) })
if len(ended) > runsSwept {
ended = ended[:runsSwept]
}
var errs []error
for _, e := range ended {
if r.now().Sub(e.at) >= r.set.kept {
errs = append(errs, r.deleteRun(e.key, func(run meshwants.CompositionRun) bool {
return run.Ended != nil && r.now().Sub(*run.Ended) >= r.set.kept
}))
}
}
return errors.Join(errs...)
}
// deleteRun deletes a run's key with its bound keys while the value it holds is still one gone judges gone, each delete at the
// revision just read and judged again on a conflict: a run that replaced the key meanwhile, and its bounds, stand.
func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionRun) bool) error {
for {
run, rev, ok, err := r.read(key)
if err != nil || !ok || !gone(run) {
return err
}
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil {
return err
}
if err := r.kv.delAt(key, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return err
}
r.heard()
return nil
}
}
// clearBounds deletes the bound keys of the run of ask while key holds that run (or none) and the bound holds that ask: each at
// the revision it was read at, read again on a conflict, so a bound a newer run of the key armed is never deleted.
func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error {
var errs []error
for _, prefix := range prefixes {
for {
run, _, ok, err := r.read(key)
if err != nil {
errs = append(errs, err)
break
}
if ok && run.Ask != ask {
break // a newer run holds the key: its bounds are its own
}
held, rev, standing, err := r.boundOf(prefix + key)
if err != nil || !standing {
errs = append(errs, err)
break
}
if held != ask {
break // the bound of another ask (A-470n): not this run's to delete
}
if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
errs = append(errs, err)
}
break
}
}
return errors.Join(errs...)
}
// stepEnded writes at as the `ended` of the reader asked under id in each run of its step's candidate naming it, by
// compare-and-set (A-470m): the time its step said its end, carried with it into a rerun's run.
func (r *compositionRuns) stepEnded(id string, step meshwants.StepRun, at time.Time) {
candidate, err := candidateKeyOf(step)
if err != nil {
r.logf("the end of the composition step %s is not written on its runs: %v", id, err)
return
}
keys, err := r.keysOf(candidate, id)
if err != nil {
r.logf("the end of the composition step %s is not written on its runs: %v", id, err)
return
}
at = at.UTC()
for _, key := range keys {
if _, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
i := slices.IndexFunc(c.Readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
if i < 0 {
return false
}
c.Readers[i].Ended = &at
return true
}); err != nil {
r.logf("the end of the composition step %s is not written on the composition run %s: %v", id, key, err)
}
}
}
// hasReader is whether readers holds the step asked under id.
func hasReader(readers []meshwants.CompositionReader, id string) bool {
return slices.ContainsFunc(readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
}
// readerOf is a composition step as a run's reader: its StepRun with its ask's id (A-468).
func readerOf(id string, s meshwants.StepRun) meshwants.CompositionReader {
return meshwants.CompositionReader{ID: id, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Branch: s.Branch, Commit: s.Commit,
Revision: s.Revision, Source: s.Source, Fingerprint: s.Fingerprint, Run: s.Run, Rerun: s.Rerun, By: s.By}
}
// stepRunOf is a reader's StepRun again, for its step-ended.
func stepRunOf(c meshwants.CompositionReader) meshwants.StepRun {
return meshwants.StepRun{Pipeline: c.Pipeline, Step: c.Step, Module: c.Module, Branch: c.Branch, Commit: c.Commit, Revision: c.Revision,
Source: c.Source, Fingerprint: c.Fingerprint, Run: c.Run, Rerun: c.Rerun, By: c.By}
}
// watchRuns hears the changes of the state composition-runs (principle 1): each change wakes the steps waiting on a run, and
// the expiry of a bound key, the bus's marker, is handed to expired (§720). After the values that stand, ready is called.
func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, expired func(key string), ready func()) error {
w, err := kv.WatchAll(ctx)
if err != nil {
return err
}
go func() {
defer func() { _ = w.Stop() }()
for {
select {
case <-ctx.Done():
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil {
if ready != nil {
ready()
ready = nil
}
continue
}
r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut)
if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil {
expired(e.Key())
}
r.heard()
}
}
}()
return nil
}
// end ends the run of ask in its key with an outcome, its words and the machine's verdict, by compare-and-set; it answers
// false when the key holds another run or the run has ended already.
func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
ended := false
_, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
at := r.now().UTC()
c.Ended, c.Outcome, c.Said, c.Verdict = &at, outcome, said, verdict
ended = true
return true
})
return ended, err
}
// keyOfAsk is the key of the run whose ask is ask, or none.
func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) {
all, err := r.keys()
if err != nil {
return "", meshwants.CompositionRun{}, err
}
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err == nil && ok && run.Ask == ask {
return key, run, nil
}
}
return "", meshwants.CompositionRun{}, nil
}
// heardEnd ends the run of the ask a build seat's step-ended names (§719, A-240): a run ask that ended other than passed
// could not run; a passed one's answer, read with read, is the machine's verdict, or why it is none. An end of an ask no run
// names is another asker's, or of a run gone, and changes nothing.
func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.StepEnded) ([]byte, error)) error {
if e.Asker != runAsker || e.Ask == "" {
return nil
}
key, run, err := r.keyOfAsk(e.Ask)
if err != nil || key == "" {
return err
}
if run.Ended != nil { // ended by its bound or a drop: the end heard later is ignored (§720)
r.logf("the step-ended of the run ask %s is ignored: its composition run %s ended before it (%s)", e.Ask, key, run.Said)
return nil
}
node := run.Machine
outcome, said := meshwants.CompositionRunOutcomeCouldNotRun, ""
var verdict map[string]any
if e.Outcome != buildstep.OutcomePassed {
said = fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, e.Ask, e.Outcome, e.Node, e.Said)
} else if answer, err := read(e); err != nil { // the program's last line of standard output (A-254)
said = node + ": " + err.Error()
} else if verdict, said = runVerdict(node, answer); said == "" {
outcome = meshwants.CompositionRunOutcomePassed
}
_, err = r.endRun(key, e.Ask, outcome, said, verdict)
return err
}
// await waits until each run of keys has ended, through the state's changes heard, or ctx ends: it answers each run as it
// stands then; a key gone could not run.
func (r *compositionRuns) await(ctx context.Context, keys []string) map[string]meshwants.CompositionRun {
out := map[string]meshwants.CompositionRun{}
for {
wake := r.wake()
open := false
for _, key := range keys {
run, _, ok, err := r.read(key)
switch {
case err != nil:
open = true
continue
case !ok:
at := r.now().UTC()
candidate, machine, _ := strings.Cut(key, ".")
run = meshwants.CompositionRun{Candidate: unescape(candidate), Machine: unescape(machine), Ended: &at,
Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: unescape(machine) + ": its composition run " + key + " is gone"}
case run.Ended == nil:
open = true
}
out[key] = run
}
if !open {
return out
}
select {
case <-ctx.Done():
return out
case <-wake:
}
}
}
// unescape is a key's part as a name again; a part that does not read stays as it is.
func unescape(part string) string {
if name, err := keys.UnescapeName(part); err == nil {
return name
}
return part
}
// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands.
func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared {
out := []meshwants.Shared{}
for _, key := range keys {
run, ok := runs[key]
if !ok {
if run, _, ok, _ = r.read(key); !ok {
continue
}
}
out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask})
}
return out
}
// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers
// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) {
all, err := r.keys()
if err != nil {
return nil, err
}
var ids []string
var errs []error
for _, key := range all {
if !isRunKey(key) {
continue
}
var left []string
var leaving []meshwants.CompositionReader
run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool {
left, leaving = nil, nil
keep := c.Readers[:0:0]
for _, reader := range c.Readers {
if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition {
left = append(left, reader.ID)
leaving = append(leaving, reader)
continue
}
keep = append(keep, reader)
}
c.Readers = keep
return len(left) > 0
})
if err != nil {
errs = append(errs, err)
continue
}
if !ok || len(left) == 0 {
continue
}
ids = append(ids, left...)
if run.Ended == nil && r.untaken(key, run.Ask) { // a reader leaving a run not yet taken: its wait ends at once (A-470u)
r.sayWaits(run, leaving, "superseded")
}
if stop != nil {
stop(left) // the steps that left say no end, the run's included
}
if run.Ended != nil || len(run.Readers) > 0 {
continue
}
why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled"
if r.drop != nil {
if err := r.drop(run.Ask, why); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil {
errs = append(errs, err)
}
}
return ids, errors.Join(errs...)
}
// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there.
func passedOn(module string, run meshwants.CompositionRun) bool {
e := judgeRun(module, run)
return e.failed == "" && e.couldNot == ""
}
// formerOf is the place in readers of a step's former ask: the reader of the same pipeline's composition step, or -1.
func formerOf(readers []meshwants.CompositionReader, pipeline string) int {
return slices.IndexFunc(readers, func(c meshwants.CompositionReader) bool {
return c.Pipeline == pipeline && c.Step == meshwants.CompositionReaderStepComposition
})
}
// rejoin is join for a step asked again (§727, A-468g, A-470d): on a machine whose ended run it passed it keeps that verdict,
// reading the run under its new ask; a running run it joins; an ended run it did not read and that could run it reads, as one
// started after its last end; otherwise it starts a new run, replacing the ended key by compare-and-set and carrying every former
// reader into the new value: each that did not pass on the former run marked `rerun` with by the rerun's caller, each that passed
// not (A-470d).
func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
key := runKeyOf(candidate, machine)
for {
run, rev, ok, err := r.read(key)
if err != nil {
return joined{}, err
}
if !ok {
return r.join(reader, candidate, machine, ask)
}
at := formerOf(run.Readers, reader.Pipeline)
ended := run.Ended != nil
keep := ended && at >= 0 && passedOn(reader.Module, run)
read := !ended || (at < 0 && run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun)
if keep || read {
if at >= 0 && run.Readers[at].ID == reader.ID {
return joined{key: key, run: run}, nil
}
var former []meshwants.CompositionReader
if at >= 0 {
former = []meshwants.CompositionReader{run.Readers[at]}
run.Readers[at] = reader
} else {
run.Readers = append(run.Readers, reader)
}
if err := r.write(key, run, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
if run.Ended == nil && r.untaken(key, run.Ask) { // as join: a reader of a run not yet taken waits for a build node (A-469y)
r.sayWaits(run, former, "superseded") // the place it takes: that wait ends at once (A-470u)
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
}
return joined{key: key, run: run}, nil
}
if err := r.room(); err != nil {
return joined{}, err
}
started := r.now().UTC()
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started}
for i, c := range run.Readers {
if i == at {
continue
}
c.Rerun, c.By = !passedOn(c.Module, run), ""
if c.Rerun {
c.By = reader.By
}
fresh.Readers = append(fresh.Readers, c)
}
fresh.Readers = append(fresh.Readers, reader)
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { // the former run's bounds go with it
r.logf("the bound keys of the former composition run %s are not deleted: %v", key, err)
}
if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
}
r.sayWaits(fresh, fresh.Readers, "")
return joined{key: key, started: true, run: fresh}, nil
}
}
// write writes a run's value at its revision.
func (r *compositionRuns) write(key string, run meshwants.CompositionRun, rev uint64) error {
raw, err := json.Marshal(run)
if err != nil {
return err
}
if _, err := r.kv.update(key, raw, rev); err != nil {
return err
}
r.heard()
return nil
}
@@ -0,0 +1,404 @@
//go:build integration
package main
import (
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"mesh-delivery-assigner/internal/buckets"
)
// Test 108 (design 47 G §733; §718 to §723, §727, §728; A-468 to A-470f): the assigner started whole against the bus composes
// each machine once per candidate. Two or three composition steps of one candidate on two machines make one key, one request and
// one run ask per machine, and each step reads both verdicts; a machine blaming a and b fails their steps and passes c's with a
// warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails
// every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound,
// one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never
// taken ends at run-untaken-bound; a bound holds its run's ask (A-470n). A rerun of a starts one run on the machine it did not
// pass and moves b; a superseded step leaves
// a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the
// readers and deletes the key; at the cap a step ends could not run, naming the fill.
func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
b := startBed(t, "")
b.settings = map[string]string{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "15s", // past a restart
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "1"}
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
const commit = "1111111111111111111111111111111111111111"
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
nodebuildgo.SubjectStepEndedAll, nodebuildgo.SubjectAskTakenAll, nodebuildgo.SubjectStepProgressAll, pipelineEndedSubject},
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second}); err != nil { // an event handed to the stopped assigner's pull comes again soon
t.Fatal(err)
}
for _, node := range []string{"n1", "n2"} {
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
t.Fatal(err)
}
}
awaitKept(t, b, nodeengine.StreamRunning, 2)
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: commit, Path: ".",
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
b.cat.mu.Lock()
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine}
for _, m := range []string{"a", "b", "c"} {
b.cat.entries[trunkBranch][m] = meshcatalog.CatalogEntry{Module: m, Fp: "src2:" + m, Repository: "novox/" + m, Commit: commit, Path: ".", Manifest: map[string]any{"module": m}}
}
b.cat.rev[trunkBranch] = 3
b.cat.mu.Unlock()
for _, node := range []string{"n1", "n2"} {
for _, m := range []string{"a", "b", "c"} {
b.place(t, node, m)
}
}
var mu sync.Mutex
inputsAsked := map[string]int{} // by candidate and node
var drops []string
respond := func(subject string, f func(m *nats.Msg) any) {
sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": f(m), "node": "n0"})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
respond("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) any {
node := strings.Split(m.Subject, ".")[5]
var in nodeengine.ComposeCandidateInput
_ = json.Unmarshal(m.Data, &in)
mu.Lock()
inputsAsked[in.Candidate+" "+node]++
mu.Unlock()
return map[string]any{"candidate": in.Candidate, "node": node, "verdict": "composed", "input": map[string]any{"node": node}, "modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}
})
respond("mesh.seat.mesh-catalog.call.entry.*", func(*nats.Msg) any { return meshcatalog.EntryAnswer{Entry: engine} })
respond("mesh.seat.node-build-go.call.drop.*", func(m *nats.Msg) any {
var in nodebuildgo.DropInput
_ = json.Unmarshal(m.Data, &in)
mu.Lock()
drops = append(drops, in.ID)
mu.Unlock()
return map[string]any{}
})
respond("mesh.seat.mesh-hand-acts.call.hand-act.*", func(*nats.Msg) any { return map[string]any{"log": "hand-act-1"} })
// The ends are read from the stream through a durable consumer, as the runner hears them: an end the bus drops as a
// repeated message id is never heard (A-470q).
ended := make(chan meshwants.StepEnded, 32)
runner, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "runner-stand-in",
FilterSubject: "mesh.seat.mesh-wants.event.step-ended.n1", AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy})
if err != nil {
t.Fatal(err)
}
heard, err := runner.Consume(func(m jetstream.Msg) {
var e meshwants.StepEnded
_ = json.Unmarshal(m.Data(), &e)
_ = m.Ack()
ended <- e
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(heard.Stop)
subscribed(t, nc)
from := b.journal.mark()
_, stop := b.startStoppable(t)
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
select {
case <-b.seatListRead:
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
queue, _ := buildstep.AskStream(buildstep.AskGo)
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants",
AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
step := func(id, module, run string, rerun bool) {
t.Helper()
s := meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: trunkBranch, Revision: 3,
Commit: commit, Fingerprint: "src2:" + module, Run: run, Rerun: rerun}
if run != "" {
s.Pipeline = "run-" + run + "." + module
}
if rerun {
s.By = "op@n1"
}
raw, _ := json.Marshal(s)
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", id)
msg.Data = raw
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
}
type asked struct {
ask buildstep.StepAsk
machine string
}
// asks takes n run asks of the stand-in build seat, each with the machine its request composes.
asks := func(n int) []asked {
t.Helper()
var out []asked
for range n {
got, err := worker.Next(jetstream.FetchMaxWait(20 * time.Second))
if err != nil {
t.Fatalf("%d of %d run asks put: %v", len(out), n, err)
}
_ = got.Ack()
a, err := buildstep.DecodeStepAsk(got.Data())
if err != nil || a.Run == nil {
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
}
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
if err != nil || len(req.Machines) != 1 || req.BlameTries != 20 {
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
}
out = append(out, asked{a, req.Machines[0]})
}
return out
}
noMoreAsks := func(what string) {
t.Helper()
if got, err := worker.Next(jetstream.FetchMaxWait(300 * time.Millisecond)); err == nil {
t.Fatalf("%s: one more run ask is put: %s", what, got.Data())
}
}
say := func(event string, node string, body any) {
t.Helper()
raw, _ := json.Marshal(body)
if _, err := b.js.Publish(b.ctx, "mesh.seat.node-build-go.event."+event+"."+node, raw); err != nil {
t.Fatal(err)
}
}
answer := func(a asked, machine string) {
t.Helper()
v := fmt.Sprintf(`{"v":%d,"cases":[{"name":"change","manifests":{},"machines":{%q:%s}}]}`, composer.Version, a.machine, machine)
say("step-ended", "b1", buildstep.StepEnded{V: 1, ID: a.ask.ID, Ask: a.ask.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: v, At: time.Now().UTC()})
}
ends := func(n int, what string) map[string]meshwants.StepEnded {
t.Helper()
out := map[string]meshwants.StepEnded{}
for len(out) < n {
select {
case e := <-ended:
out[e.ID] = e
case <-time.After(20 * time.Second):
t.Fatalf("%s: %d of %d steps ended: %+v", what, len(out), n, out)
}
}
return out
}
runKeys := func(candidate string) []string {
all, _ := b.buckets[buckets.CompositionRuns()].Keys(b.ctx)
var out []string
for _, k := range all {
if isRunKey(k) && strings.HasPrefix(k, keysPart(candidate)+".") {
out = append(out, k)
}
}
return out
}
// readers waits, through a watch of the key, until the run of key has n readers.
readers := func(key string, n int) {
t.Helper()
w, err := b.buckets[buckets.CompositionRuns()].Watch(b.ctx, key)
if err != nil {
t.Fatal(err)
}
defer func() { _ = w.Stop() }()
deadline := time.After(10 * time.Second)
for {
select {
case e := <-w.Updates():
var run meshwants.CompositionRun
if e != nil && json.Unmarshal(e.Value(), &run) == nil && len(run.Readers) == n {
return
}
case <-deadline:
t.Fatalf("the run %s has not %d readers", key, n)
}
}
}
blames := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and b's"},{"module":"b","problem":"/etc/x is a's and b's"}]}`
composes := `{"composes":true,"blamed":[]}`
// One candidate, three steps, two machines: two runs.
for _, m := range []string{"a", "b", "c"} {
step("step-"+m, m, "", false)
}
first := asks(2)
noMoreAsks("three steps of one candidate")
for _, a := range first {
answer(a, map[string]string{"n1": composes, "n2": blames}[a.machine])
}
got := ends(3, "the three steps")
if a, bb, c := got["step-a"], got["step-b"], got["step-c"]; a.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(a.Said, "n2 holds a") ||
bb.Outcome != meshwants.StepEndedOutcomeFailed || c.Outcome != meshwants.StepEndedOutcomePassed || len(c.Warnings) != 2 ||
!strings.Contains(c.Warnings[0], "n2 holds a") || len(c.Shared) != 2 || c.Shared[0].Ask == c.Shared[1].Ask {
t.Fatalf("the steps of one candidate: %+v", got)
}
if keys := runKeys("main.3"); len(keys) != 2 || inputsAsked["main.3 n1"] != 1 || inputsAsked["main.3 n2"] != 1 {
t.Fatalf("keys %v, inputs asked %v: want one run and one input per machine", keys, inputsAsked)
}
// Another candidate (a run on request): its own runs; an old composer and a machine blaming none fail.
step("step-r1", "a", "r1", false)
for _, a := range asks(2) {
answer(a, map[string]string{"n1": `{"composes":false,"reason":"x"}`, "n2": `{"composes":false,"reason":"own","blamed":[]}`}[a.machine])
}
if e := ends(1, "another candidate")["step-r1"]; e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose and blames no module") ||
len(e.Warnings) != 1 || e.Warnings[0] != "n1: the composer names no culprits" {
t.Fatalf("a step of another candidate: %+v", e)
}
// The bound: n1's ask taken and saying progress past the bound, then silent; n2's never taken.
from = b.journal.mark()
step("step-r2", "a", "r2", false)
bound := asks(2)
var n1 asked
for _, a := range bound {
if a.machine == "n1" {
n1 = a
}
}
say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"})
for i := range 3 { // past the 2 s bound, each beat within it
select {
case e := <-ended:
t.Fatalf("a run saying its progress ended: %+v", e)
case <-time.After(time.Second):
}
if i == 0 { // the bound holds its run's ask, published at the key's latest revision with its own time to live (A-470n)
runs := b.buckets[buckets.CompositionRuns()]
all, _ := runs.Keys(b.ctx)
held := ""
for _, k := range all {
if strings.HasPrefix(k, runBoundPrefix) {
if e, err := runs.Get(b.ctx, k); err == nil {
var v boundValue
_ = json.Unmarshal(e.Value(), &v)
held += v.Ask
}
}
}
if held != n1.ask.ID {
t.Fatalf("the bound keys hold the asks %q, want n1's %s alone", held, n1.ask.ID)
}
}
say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"})
}
e := ends(1, "the bounds")["step-r2"]
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "said no progress within 2s (composition-step-bound)") ||
!strings.Contains(e.Said, "taken by no build node within 15s (run-untaken-bound)") {
t.Fatalf("past the bounds: %+v", e)
}
mu.Lock()
dropped := len(drops)
mu.Unlock()
if dropped != 2 {
t.Fatalf("%d asks dropped past their bounds, want 2", dropped)
}
answer(n1, composes) // heard after the bound: ignored and journaled
b.journal.await(t, from, "the step-ended of the run ask "+n1.ask.ID+" is ignored", "a late step-ended", 10*time.Second)
// A rerun of a: n1 it passed keeps its verdict; n2 it failed starts one run carrying b, which moves.
step("step-a2", "a", "", true)
again := asks(1)
noMoreAsks("a rerun of a")
if again[0].machine != "n2" {
t.Fatalf("the rerun composes %s, want only the machine a did not pass", again[0].machine)
}
answer(again[0], composes)
moved := ends(2, "the rerun")
if a2, bb := moved["step-a2"], moved["step-b"]; a2.Outcome != meshwants.StepEndedOutcomePassed || bb.Outcome != meshwants.StepEndedOutcomePassed ||
!bb.Rerun || bb.By != "op@n1" || len(bb.Shared) != 2 || bb.Shared[1].Ask != again[0].ask.ID {
t.Fatalf("after the rerun (asked again %s): a2 %+v; b %+v", again[0].ask.ID, moved["step-a2"].Shared, moved["step-b"])
}
// A superseded step leaves the runs; its other reader goes on.
step("step-r3a", "a", "r3", false)
step("step-r3b", "b", "r3", false)
sup := asks(2)
readers(runKeyOf("run-r3", "n1"), 2)
readers(runKeyOf("run-r3", "n2"), 2)
pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Module: "a", Branch: trunkBranch, Commit: commit, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Steps: []mpr.StepResult{}})
if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil {
t.Fatal(err)
}
for _, a := range sup {
answer(a, composes)
}
if got := ends(1, "the superseded run"); got["step-r3b"].Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("the other reader: %+v", got)
}
select {
case e := <-ended:
t.Fatalf("a superseded step said its end: %+v", e)
case <-time.After(200 * time.Millisecond):
}
// Restarted during a run: the new assigner takes it up by its ask.
step("step-r4", "c", "r4", false)
restart := asks(2)
stop()
for _, a := range restart {
answer(a, composes)
}
from = b.journal.mark()
b.startStoppable(t)
b.journal.await(t, from, "is taken up again by its runs (ask step-r4)", "the take-up", 30*time.Second)
if e := ends(1, "the take-up")["step-r4"]; e.Outcome != meshwants.StepEndedOutcomePassed || len(e.Shared) != 2 {
t.Fatalf("the step taken up: %+v", e)
}
// compositions-drop: the readers ended naming the drop, the key deleted.
step("step-r5", "a", "r5", false)
asks(2)
body, _ := json.Marshal(meshwants.CompositionsDropInput{Candidate: "run-r5", Machine: "n1", Why: "stuck"})
reply, err := nc.Request("mesh.seat.mesh-wants.call.compositions-drop.n1~mesh-delivery-pipeline", body, 10*time.Second)
if err != nil || !strings.Contains(string(reply.Data), `"ended":[`) {
t.Fatalf("compositions-drop answered %s %v", reply.Data, err)
}
if e := ends(1, "the drop")["step-r5"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by hand: stuck") {
t.Fatalf("a reader of the dropped run: %+v", e)
}
if keys := runKeys("run-r5"); len(keys) != 1 {
t.Errorf("the keys of run-r5 after the drop: %v, want n2's alone", keys)
}
// At the cap of 1 MiB no run starts: the step ends could not run, naming the fill.
for i := range 2 {
if _, err := b.buckets[buckets.CompositionRuns()].Put(b.ctx, fmt.Sprintf("fill%d", i), make([]byte, 700<<10)); err != nil {
t.Fatal(err)
}
}
step("step-r6", "a", "r6", false)
if e := ends(1, "the cap")["step-r6"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "is at its cap of 1 MiB") {
t.Fatalf("at the cap: %+v", e)
}
}
@@ -0,0 +1,642 @@
package main
import (
"context"
"encoding/json"
"fmt"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)
// runsMem is the state composition-runs in memory, its bound keys' times to live kept beside.
type runsMem struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
ttl map[string]time.Duration
bytes int64
n uint64
// beforeDel, when set, runs once before the next delete: another write landing between a read and the delete.
beforeDel func()
// beforeArm, when set, runs once before the next arm: another write landing between a read and the arm.
beforeArm func()
}
func (k *runsMem) another() {
k.mu.Lock()
f := k.beforeDel
k.beforeDel = nil
k.mu.Unlock()
if f != nil {
f()
}
}
func newRunsMem() *runsMem {
return &runsMem{vals: map[string][]byte{}, revs: map[string]uint64{}, ttl: map[string]time.Duration{}}
}
func (k *runsMem) get(key string) ([]byte, uint64, error) {
k.mu.Lock()
defer k.mu.Unlock()
return k.vals[key], k.revs[key], nil
}
func (k *runsMem) create(key string, v []byte) (uint64, error) {
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; ok {
return 0, errRunChanged
}
k.n++
k.vals[key], k.revs[key] = v, k.n
return k.n, nil
}
func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) {
k.mu.Lock()
defer k.mu.Unlock()
if k.revs[key] != rev {
return 0, errRunChanged
}
k.n++
k.vals[key], k.revs[key] = v, k.n
return k.n, nil
}
func (k *runsMem) delAt(key string, rev uint64) error {
k.another()
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; !ok {
return nil
}
if k.revs[key] != rev {
return errRunChanged
}
delete(k.vals, key)
delete(k.revs, key)
delete(k.ttl, key)
return nil
}
func (k *runsMem) keys() ([]string, error) {
k.mu.Lock()
defer k.mu.Unlock()
var out []string
for key := range k.vals {
out = append(out, key)
}
slices.Sort(out)
return out, nil
}
func (k *runsMem) arm(key, ask string, ttl time.Duration) (uint64, error) {
k.mu.Lock()
f := k.beforeArm
k.beforeArm = nil
k.mu.Unlock()
if f != nil {
f()
}
raw, _ := json.Marshal(boundValue{Bound: key, Ask: ask})
k.mu.Lock()
defer k.mu.Unlock()
k.n++
k.vals[key], k.revs[key], k.ttl[key] = raw, k.n, ttl
return k.n, nil
}
// expire ends a bound key's time to live: the key gone, as the bus's marker leaves it.
func (k *runsMem) expire(key string) {
k.mu.Lock()
defer k.mu.Unlock()
delete(k.vals, key)
delete(k.revs, key)
delete(k.ttl, key)
}
func (k *runsMem) used() (int64, error) {
k.mu.Lock()
defer k.mu.Unlock()
if k.bytes > 0 {
return k.bytes, nil
}
var n int64
for _, v := range k.vals {
n += int64(len(v))
}
return n, nil
}
// Design 47 G §718, §719, §720, A-468, A-469d, A-469x: a run's key is `<candidate key>.<machine>`, each part escaped; two
// steps of one candidate joining at once start one run and the second reads it; a run that could not run is replaced; at the
// cap no run starts, naming the fill; a run is deleted composition-runs-kept after its end; the settings are read from their files.
func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
step := func(id, module string) meshwants.CompositionReader {
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: "main", Revision: 3})
}
t.Run("key", func(t *testing.T) {
if k := runKeyOf("main.3", "n1"); k != "main=2e3.n1" {
t.Errorf("the key is %q, want each part escaped as a key's name", k)
}
if isRunKey(runBoundPrefix+"main=2e3.n1") || isRunKey(runUntakenPrefix+"main=2e3.n1") || !isRunKey("main=2e3.n1") {
t.Error("a bound key is taken for a run's or a run's for a bound key")
}
})
t.Run("one run for two steps at once", func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
var wg sync.WaitGroup
got := make([]joined, 2)
for i, m := range []string{"a", "b"} {
wg.Add(1)
go func() {
defer wg.Done()
j, err := r.join(step("ask-"+m, m), "main.3", "n1", "run-"+m)
if err != nil {
t.Error(err)
}
got[i] = j
}()
}
wg.Wait()
runKeys := func() []string {
all, _ := kv.keys()
return slices.DeleteFunc(all, func(k string) bool { return !isRunKey(k) })
}
if got[0].started == got[1].started {
t.Fatalf("started %v and %v: want exactly one step to start the run", got[0].started, got[1].started)
}
keys := runKeys()
run, _, _, _ := r.read(runKeyOf("main.3", "n1"))
if len(keys) != 1 || len(run.Readers) != 2 || run.Candidate != "main.3" || run.Machine != "n1" || run.Started == nil {
t.Errorf("keys %v, run %+v: want one key read by both", keys, run)
}
if j, _ := r.join(step("ask-a", "a"), "main.3", "n1", "run-c"); j.started || len(j.run.Readers) != 2 {
t.Errorf("a reader joining again is added again or starts a run: %+v", j)
}
})
t.Run("could not run is never read", func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
key := runKeyOf("main.3", "n1")
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomeCouldNotRun
return true
})
j, err := r.join(step("ask-b", "b"), "main.3", "n1", "run-2")
if err != nil || !j.started || j.run.Ask != "run-2" || len(j.run.Readers) != 1 || j.run.Ended != nil {
t.Errorf("after a could not run: %+v %v, want a new run read by its starter alone", j, err)
}
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomePassed
return true
})
if j, _ := r.join(step("ask-c", "c"), "main.3", "n1", "run-3"); j.started || j.run.Ask != "run-2" {
t.Errorf("a passed run is not read: %+v", j)
}
if j, _ := r.join(step("ask-d", "d"), "main.4", "n1", "run-4"); !j.started {
t.Errorf("a step of another candidate reads the first's run: %+v", j)
}
})
t.Run("cap", func(t *testing.T) {
kv := newRunsMem()
kv.bytes = 16 << 20
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
_, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1")
if err == nil || !strings.Contains(err.Error(), "at its cap of 16 MiB (16777216 bytes kept)") {
t.Errorf("at the cap: %v, want the fill named", err)
}
})
t.Run("kept", func(t *testing.T) {
kv := newRunsMem()
at := now
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return at }, t.Logf)
for _, c := range []string{"main.1", "main.2"} {
if _, err := r.join(step("ask-"+c, "a"), c, "n1", "run-"+c); err != nil {
t.Fatal(err)
}
}
old := now.Add(-25 * time.Hour)
_, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true })
_, _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), "run-main.1", time.Minute)
recent := now.Add(-time.Hour)
_, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true })
if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil {
t.Fatal(err)
}
keys, _ := kv.keys()
for _, k := range []string{runBoundPrefix, runUntakenPrefix} {
if v, _, _ := kv.get(k + runKeyOf("main.1", "n1")); v != nil {
t.Errorf("the key %s stays past its run", k+runKeyOf("main.1", "n1"))
}
}
keys = slices.DeleteFunc(keys, func(k string) bool { return !isRunKey(k) })
if want := []string{runKeyOf("main.2", "n1"), runKeyOf("main.9", "n1")}; !slices.Equal(keys, want) {
t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want)
}
})
t.Run("a listing that misses a key being written", func(t *testing.T) {
kv := &missingRuns{runsMem: newRunsMem()}
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
kv.hide = runKeyOf("main.3", "n1")
if key, _, _ := r.keyOfAsk("run-1"); key != "" {
t.Fatalf("the stand's listing does not hide the key")
}
r.saw(kv.hide, true) // as the state's watch hands it
if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide {
t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key)
}
r.saw(kv.hide, false)
if keys, _ := r.keys(); slices.Contains(keys, kv.hide) {
t.Errorf("a key the watch saw deleted is still listed: %v", keys)
}
})
// No delete lands on a value newer than the one it was judged from (the kind test 109 met in test-runs): a rerun's run
// replacing the key, and its untaken bound, stand.
for _, c := range []struct {
name string
ended time.Time
act func(r *compositionRuns, key string) error
}{
{"the sweep of a run replaced meanwhile", now.Add(-25 * time.Hour), func(r *compositionRuns, _ string) error { return r.sweep() }},
{"the end of a run replaced meanwhile", time.Time{}, func(r *compositionRuns, key string) error {
_, err := r.endRun(key, "run-1", meshwants.CompositionRunOutcomePassed, "", nil)
return err
}},
} {
t.Run(c.name, func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
key := runKeyOf("main.3", "n1")
if !c.ended.IsZero() {
_, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true })
}
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound
for {
raw, rev, _ := kv.get(key)
var run meshwants.CompositionRun
_ = json.Unmarshal(raw, &run)
if run.Ended == nil {
at := now
run.Ended, run.Outcome = &at, "failed"
}
fresh, _ := json.Marshal(meshwants.CompositionRun{Candidate: "main.3", Machine: "n1", Ask: "run-2", Started: &now, Readers: run.Readers})
if _, err := kv.update(key, fresh, rev); err == nil {
break
}
}
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
}
if err := c.act(r, key); err != nil {
t.Fatal(err)
}
if run, _, ok, _ := r.read(key); !ok || run.Ask != "run-2" || run.Ended != nil {
t.Errorf("the key holds %+v (stands %v), want the rerun's running run-2", run, ok)
}
if v, _, _ := kv.get(runUntakenPrefix + key); v == nil {
t.Error("the untaken bound the rerun's run armed is deleted")
}
})
}
t.Run("settings", func(t *testing.T) {
dir := t.TempDir()
files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h",
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "", "MESH_ASSIGNER_BLAME_TRIES_FILE": "5"}
env := map[string]string{}
for name, content := range files {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
env[name] = p
}
s, err := runSettingsOf(func(k string) string { return env[k] })
if err != nil || s.capBytes != 8<<20 || s.kept != 2*time.Hour || s.stepBound != 2*time.Second || s.untaken != 24*time.Hour || s.blameTries != 5 {
t.Errorf("settings %+v %v", s, err)
}
_ = os.WriteFile(env["MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE"], []byte("30"), 0o600)
if _, err := runSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "not a positive Go duration") {
t.Errorf("a bound of no unit: %v", err)
}
if s, err := runSettingsOf(func(string) string { return "" }); err != nil || s != defaultRunSettings {
t.Errorf("no files: %+v %v, want the defaults", s, err)
}
})
raw, _ := json.Marshal(step("x", "a"))
if !strings.Contains(string(raw), `"id":"x"`) {
t.Errorf("a reader names no ask id: %s", raw)
}
}
// Design 47 G §720, A-466s, A-468b, A-469y, A-470: a run's untaken bound is armed at its start and each reader said waiting for a
// build node; ask-taken arms the bound of composition-step-bound and ends the waits ran; each step-progress arms it again; the
// bound's expiry drops the ask and ends the run could not run naming the ask and the bound, a step-ended heard later ignored and
// journaled; the untaken bound's expiry ends it naming run-untaken-bound, the waits superseded; the readers of a run with no end
// are taken up at the start.
func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
reader := func(id string) meshwants.CompositionReader {
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id, Branch: "main", Revision: 3})
}
type said struct{ event, how, module string }
setup := func(t *testing.T) (*runsMem, *compositionRuns, *[]said, *[]string, *[]string) {
kv := newRunsMem()
var saids []said
var drops, journal []string
var mu sync.Mutex
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, func(f string, a ...any) {
mu.Lock()
journal = append(journal, fmt.Sprintf(f, a...))
mu.Unlock()
})
r.say = func(event string, body any, _ string) error {
w := body.(meshwants.Wait)
if w.On != waitOnBuildNode || w.Step != "pipeline" || w.Owner != meshwants.Seat || w.Node == "" {
t.Errorf("the wait said %+v", w)
}
saids = append(saids, said{event, w.How, w.Module})
return nil
}
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
return kv, r, &saids, &drops, &journal
}
key := runKeyOf("main.3", "n1")
t.Run("taken, progress, bound", func(t *testing.T) {
kv, r, saids, drops, journal := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
if kv.ttl[runUntakenPrefix+key] != 24*time.Hour || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitHeld, "", "b"}}) {
t.Fatalf("at the start: untaken %v, said %v", kv.ttl, *saids)
}
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: "someone-else"}); err != nil || kv.ttl[runBoundPrefix+key] != 0 {
t.Fatalf("another asker's ask-taken arms a bound: %v %v", kv.ttl, err)
}
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
t.Fatal(err)
}
_, untaken, _ := kv.get(runUntakenPrefix + key)
if kv.ttl[runBoundPrefix+key] != 30*time.Minute || untaken != 0 || len(*saids) != 4 || (*saids)[2].how != "ran" || (*saids)[3].how != "ran" {
t.Fatalf("at ask-taken: %v, said %v", kv.ttl, *saids)
}
before := kv.revs[runBoundPrefix+key]
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before {
t.Fatalf("a step-progress does not arm the bound again: %v", err)
}
kv.expire(runBoundPrefix + key)
r.expired(runBoundPrefix + key)
run, _, _, _ := r.read(key)
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" ||
len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") {
t.Fatalf("past the bound: %+v, drops %v", run, *drops)
}
if got := judgeRun("a", run); got.couldNot != run.Said {
t.Errorf("a reader of the run past its bound: %+v", got)
}
late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"}
if err := r.heardEnd(late, func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
t.Fatal(err)
}
if again, _, _, _ := r.read(key); again.Said != run.Said || !slices.ContainsFunc(*journal, func(l string) bool {
return strings.Contains(l, "step-ended of the run ask run-1 is ignored") && strings.Contains(l, key)
}) {
t.Errorf("a step-ended after the bound: %+v, journal %v", again, *journal)
}
if keys, _ := kv.keys(); !slices.Equal(keys, []string{key}) {
t.Errorf("the bound keys stay past the run's end: %v", keys)
}
})
t.Run("untaken", func(t *testing.T) {
kv, r, saids, drops, _ := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
kv.expire(runUntakenPrefix + key)
r.expired(runUntakenPrefix + key)
run, _, _, _ := r.read(key)
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" ||
len(*drops) != 1 || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitEnded, "superseded", "a"}}) {
t.Errorf("untaken past its bound: %+v, drops %v, said %v", run, *drops, *saids)
}
r.expired(runUntakenPrefix + key) // a run ended takes nothing
if len(*drops) != 1 {
t.Errorf("an ended run is dropped again: %v", *drops)
}
})
// A-470n, A-470o: a bound holds its run's ask, armed for it alone; a late signal of a run replaced moves nothing and is
// journaled; an expiry ends the run only while its bound is still absent and the run taken (bound.) or untaken (untaken.).
rerun := func(kv *runsMem, r *compositionRuns) func() { // a rerun replaces run-1 by run-2, its bounds gone, run-2's untaken armed
return func() {
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { c.Ask = "run-2"; return true })
kv.expire(runBoundPrefix + key)
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
}
}
taken := func(t *testing.T, r *compositionRuns) {
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
t.Fatal(err)
}
}
for _, c := range []struct {
name string
act func(t *testing.T, kv *runsMem, r *compositionRuns)
ended bool
bound string // the ask the bound key holds, "" for none
untaken string
journal string
}{
{"a late ask-taken of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
kv.beforeArm = rerun(kv, r)
taken(t, r)
}, false, "", "run-2", "the ask-taken of the run ask run-1 is late"},
{"a late step-progress of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
kv.beforeArm = rerun(kv, r)
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
t.Fatal(err)
}
kv.expire(runBoundPrefix + key) // a bound left standing would end run-2 here
r.expired(runBoundPrefix + key)
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
{"a step-progress of a run gone", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
rerun(kv, r)()
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
t.Fatal(err)
}
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
{"an expiry heard after the bound was armed again", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
r.expired(runBoundPrefix + key)
}, false, "run-1", "", "the expiry of " + runBoundPrefix + key + " ends nothing"},
{"the bound's expiry of a run not taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
kv.expire(runBoundPrefix + key)
r.expired(runBoundPrefix + key)
}, false, "", "run-1", "the expiry of " + runBoundPrefix + key + " ends nothing"},
{"the untaken expiry of a run taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
kv.expire(runUntakenPrefix + key)
r.expired(runUntakenPrefix + key)
}, false, "run-1", "", "the expiry of " + runUntakenPrefix + key + " ends nothing"},
{"an end of another ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
if ended, err := r.end(key, "run-0", meshwants.CompositionRunOutcomePassed, "", nil); ended || err != nil {
t.Errorf("the end of run-0 ended the run of run-1: %v %v", ended, err)
}
}, false, "", "run-1", ""},
{"the ask-taken of the run's own ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
}, false, "run-1", "", ""},
} {
t.Run(c.name, func(t *testing.T) {
kv, r, _, _, journal := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
c.act(t, kv, r)
run, _, _, _ := r.read(key)
if (run.Ended != nil) != c.ended {
t.Errorf("the run ended %v (%s), want %v", run.Ended != nil, run.Said, c.ended)
}
for prefix, want := range map[string]string{runBoundPrefix: c.bound, runUntakenPrefix: c.untaken} {
if held, _, _, _ := r.boundOf(prefix + key); held != want {
t.Errorf("%s holds the ask %q, want %q", prefix+key, held, want)
}
}
if c.journal != "" && !slices.ContainsFunc(*journal, func(l string) bool { return strings.Contains(l, c.journal) }) {
t.Errorf("the journal %v does not say %q", *journal, c.journal)
}
})
}
// A-469y, A-470u: a step asked again that joins a running run not yet taken, beside a reader or in its pipeline's former
// reader's place, says its own wait held, as join does; the former reader's wait, and that of a reader leaving, ends at once
// superseded.
held := func(m string) said { return said{meshwants.EventWaitHeld, "", m} }
gone := func(m string) said { return said{meshwants.EventWaitEnded, "superseded", m} }
for _, c := range []struct {
name string
act func(t *testing.T, r *compositionRuns)
want []said
}{
{"a rerun joins", func(t *testing.T, r *compositionRuns) {
again := reader("b")
again.Rerun = true
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
}, []said{held("a"), held("b")}},
{"a rerun takes its former place", func(t *testing.T, r *compositionRuns) {
again := reader("a2")
again.Pipeline, again.Module, again.Rerun = "main.3.a", "a2", true
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
}, []said{held("a"), gone("a"), held("a2")}},
{"a reader leaves", func(t *testing.T, r *compositionRuns) {
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
if _, err := r.leave("main.3.a", nil); err != nil {
t.Fatal(err)
}
}, []said{held("a"), held("b"), gone("a")}},
} {
t.Run(c.name, func(t *testing.T) {
_, r, saids, _, _ := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
c.act(t, r)
if !slices.Equal(*saids, c.want) {
t.Errorf("said %v, want %v", *saids, c.want)
}
})
}
t.Run("taken up at the start", func(t *testing.T) {
_, r, _, _, _ := setup(t)
for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} {
if _, err := r.join(reader(c.id), "main.3", c.machine, "run-"+c.machine); err != nil {
t.Fatal(err)
}
}
ended := now
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &ended, "passed"; return true })
// A rerun's run carrying readers whose steps said their end (A-470l, A-470m): c passed and is not taken up, d is marked rerun.
_, _, _ = r.change(runKeyOf("main.3", "n1"), func(run *meshwants.CompositionRun) bool {
for _, id := range []string{"c", "d"} {
c := reader(id)
c.Ended, c.Rerun = &ended, id == "d"
run.Readers = append(run.Readers, c)
}
return true
})
readers, err := r.readersOfOpen()
if err != nil || len(readers) != 3 || readers[0].ID != "a" || readers[1].ID != "b" || readers[2].ID != "d" {
t.Fatalf("the readers taken up: %+v %v", readers, err)
}
if keys, _ := r.keysOf("main.3", "a"); !slices.Equal(keys, []string{runKeyOf("main.3", "n1"), runKeyOf("main.3", "n2")}) {
t.Errorf("the runs of reader a: %v", keys)
}
// A-470q: a reader with no `ended` says its first end under its ask's id; d, marked rerun, says its end again under
// `<id>.<the open run's ask>`.
ids := make(chan [2]string, 3)
steps := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf,
resume: func(context.Context, string, meshwants.StepRun) stepOutcome {
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
},
say: func(_ string, body any, id string) error {
ids <- [2]string{body.(meshwants.StepEnded).ID, id}
return nil
}}
steps.takeUp(context.Background(), r)
got := map[string]string{}
for range 3 {
select {
case e := <-ids:
got[e[0]] = e[1]
case <-time.After(5 * time.Second):
t.Fatalf("the ends of the take-up: %v", got)
}
}
if want := map[string]string{"a": "a", "b": "b", "d": "d.run-n1"}; !maps.Equal(got, want) {
t.Errorf("the message ids of the take-up's ends: %v, want %v", got, want)
}
})
}
// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment.
type missingRuns struct {
*runsMem
hide string
}
func (k *missingRuns) keys() ([]string, error) {
all, err := k.runsMem.keys()
return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err
}
@@ -0,0 +1,162 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The composition runs' read and drop (design 47 G §728; A-469f, A-469u, A-469v): the seat mesh-wants' verbs `compositions`
// (class read) and `compositions-drop` (class act, §391's default callers), answered by the assigner's process, which keeps the
// runs and says the readers' ends; the module's tools `assigner_compositions` and `assigner_compositions_drop` call them.
// The tools that call the two verbs.
const (
compositionsTool = "assigner_compositions"
compositionsDropTool = "assigner_compositions_drop"
)
// compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the
// machine named only those, and the state's fill against its cap.
func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) {
all, err := r.keys()
if err != nil {
return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err)
}
slices.Sort(all)
out := meshwants.CompositionsAnswer{Runs: []meshwants.CompositionRun{}}
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err != nil || !ok || (in.Candidate != "" && run.Candidate != in.Candidate) || (in.Machine != "" && run.Machine != in.Machine) {
continue
}
run.Key = key
out.Runs = append(out.Runs, run)
}
out.Fill, err = r.fill()
return out, err
}
// drop drops one composition run by hand (§728): the act and why written in the hand-act log first (record), then a running run's
// ask dropped, each of its readers' waits stopped and, for each reader whose end is owed (no `ended`, or marked rerun), its end
// said could not run naming the drop, once (A-470r), and the key deleted with its bound keys. It answers the entry deleted and
// the readers it ended.
func (c *compositionSteps) drop(r *compositionRuns, record func(meshhandactsclient.Request) error, caller string, in meshwants.CompositionsDropInput) (meshwants.CompositionsDropAnswer, error) {
why := strings.TrimSpace(in.Why)
if in.Candidate == "" || in.Machine == "" || why == "" {
return meshwants.CompositionsDropAnswer{}, errors.New("name the candidate, the machine and why the composition run is dropped")
}
key := runKeyOf(in.Candidate, in.Machine)
run, _, ok, err := r.read(key)
if err != nil {
return meshwants.CompositionsDropAnswer{}, err
}
if !ok {
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("no composition run of %s on %s is kept", in.Candidate, in.Machine)
}
if err := record(meshhandactsclient.Request{What: meshwants.VerbCompositionsDrop + " " + key, Why: why + " (" + caller + ")", Cause: "compositions-drop"}); err != nil {
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
}
said := fmt.Sprintf("%s: the composition run %s was dropped by hand: %s", in.Machine, key, why)
answer := meshwants.CompositionsDropAnswer{Run: run}
answer.Run.Key = key
if run.Ended == nil {
if r.drop != nil {
if err := r.drop(run.Ask, said); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
ids := make([]string, 0, len(run.Readers))
for _, reader := range run.Readers {
ids = append(ids, reader.ID)
}
c.leave(ids) // their waits say no end of their own
// The run ends without moving its readers: the drop says each owed end itself, once (A-470r).
if _, err := r.endRunMoving(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, r.untaken(key, run.Ask), false); err != nil {
return answer, err
}
shared := []meshwants.Shared{{Machine: run.Machine, Run: key, Ask: run.Ask}}
for _, reader := range run.Readers {
if reader.Ended != nil && !reader.Rerun {
continue // it ended passed already and keeps its verdict (A-470r)
}
again := "" // a reader whose step said its end already says it again under the dropped run's ask (A-470q)
if reader.Ended != nil {
again = run.Ask
}
c.sayEnd(reader.ID, stepRunOf(reader), nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said, Shared: shared}, again)
answer.Ended = append(answer.Ended, reader)
}
}
// The dropped run's key goes while it holds that run: a run started on it since stands.
err = r.deleteRun(key, func(c meshwants.CompositionRun) bool { return c.Ask == run.Ask })
r.heard()
return answer, err
}
// compositionVerbs are the process's handlers of the seat mesh-wants' verbs compositions and compositions-drop.
func compositionVerbs(r *compositionRuns, c *compositionSteps, record func(meshhandactsclient.Request) error) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
meshwants.VerbCompositions: func(_ string, raw json.RawMessage) (any, error) {
var in meshwants.CompositionsInput
if len(raw) > 0 {
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
}
return r.compositions(in)
},
meshwants.VerbCompositionsDrop: func(caller string, raw json.RawMessage) (any, error) {
var in meshwants.CompositionsDropInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
return c.drop(r, record, caller, in)
},
}
}
// compositionsTools are the module's tools of §728's names, each calling its verb (A-469f).
func compositionsTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
call := func(verb string, args map[string]any, names ...string) (any, error) {
in := map[string]any{}
for _, n := range names {
if v := arg(args, n); v != "" {
in[n] = v
}
}
raw, err := ask("seat:"+meshwants.Seat+"."+verb, in)
if err != nil {
return nil, err
}
var out any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("%s answered what is not JSON: %w", verb, err)
}
return out, nil
}
return []stdio.Tool{
{Name: compositionsTool, Description: "The composition runs the assigner keeps, one per candidate and machine (design 47 G §719, §728): each with " +
"its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those. Read-only.",
Input: schema(map[string]string{"candidate": "the candidate's key: <branch>.<revision>, or run-<id>", "machine": "a machine"}, nil),
Run: func(args map[string]any) (any, error) {
return call(meshwants.VerbCompositions, args, "candidate", "machine")
}},
{Name: compositionsDropTool, Description: "Drop one composition run by hand (design 47 G §728): written in the hand-act log first, then its ask " +
"is dropped, each reader ended could not run naming the drop, and the key deleted.",
Input: schema(map[string]string{"candidate": "the candidate's key", "machine": "the machine", "why": "why the run is dropped"}, []string{"candidate", "machine", "why"}),
Run: func(args map[string]any) (any, error) {
return call(meshwants.VerbCompositionsDrop, args, "candidate", "machine", "why")
}},
}
}
@@ -0,0 +1,91 @@
package main
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G §728, A-469f, A-469u, A-469v: compositions answers each run with its key and the fill, only those of a candidate or
// machine when named; compositions-drop writes the hand act first (nothing is dropped when it is not recorded), drops a running
// run's ask, ends each reader could not run naming the drop and deletes the key with its bound keys; the tools call the verbs.
func TestTheCompositionRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
var drops []string
r.drop = func(ask, why string) error { drops = append(drops, ask); return nil }
ends := newSaidEvents()
var endIDs []string // the message id of each end said
c := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf, say: func(event string, body any, id string) error {
endIDs = append(endIDs, id)
return ends.say(event, body, id)
}}
moves := 0
move := func(meshwants.CompositionRun) { moves++ }
r.moved.Store(&move)
for _, m := range []string{"n1", "n2"} {
if _, err := r.join(readerOf("ask-a", meshwants.StepRun{Pipeline: "main.3.a", Step: meshwants.StepRunStepComposition, Module: "a"}), "main.3", m, "run-"+m); err != nil {
t.Fatal(err)
}
}
// A-470r: n2's run carries a reader that ended passed (p) and one marked rerun (q); only q's end is owed, said once.
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(run *meshwants.CompositionRun) bool {
for _, id := range []string{"ask-p", "ask-q"} {
c := readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id})
c.Ended, c.Rerun = &now, id == "ask-q"
run.Readers = append(run.Readers, c)
}
return true
})
var recorded []meshhandactsclient.Request
record := func(q meshhandactsclient.Request) error { recorded = append(recorded, q); return nil }
verbs := compositionVerbs(r, c, record)
out, err := verbs[meshwants.VerbCompositions]("op", json.RawMessage(`{"machine":"n2"}`))
read, _ := out.(meshwants.CompositionsAnswer)
if err != nil || len(read.Runs) != 1 || read.Runs[0].Key != runKeyOf("main.3", "n2") || read.Runs[0].Ask != "run-n2" || len(read.Runs[0].Readers) != 3 ||
read.Fill.Store != "composition-runs" || read.Fill.Cap != 16<<20 || read.Fill.Bytes == 0 {
t.Fatalf("compositions = %+v %v", out, err)
}
refuse := compositionVerbs(r, c, func(meshhandactsclient.Request) error { return errors.New("no holder") })
if _, err := refuse[meshwants.VerbCompositionsDrop]("op", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`)); err == nil ||
!strings.Contains(err.Error(), "nothing was dropped") || len(drops) != 0 {
t.Fatalf("a drop not recorded: %v, drops %v", err, drops)
}
out, err = verbs[meshwants.VerbCompositionsDrop]("op@n1", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`))
dropped, _ := out.(meshwants.CompositionsDropAnswer)
if err != nil || dropped.Run.Key != runKeyOf("main.3", "n2") || len(dropped.Ended) != 2 || dropped.Ended[0].ID != "ask-a" || dropped.Ended[1].ID != "ask-q" ||
moves != 0 || len(endIDs) != 2 || endIDs[0] != "ask-a" || endIDs[1] != "ask-q.run-n2" ||
len(recorded) != 1 || recorded[0].Why != "stuck (op@n1)" || len(drops) != 1 || drops[0] != "run-n2" {
t.Fatalf("compositions-drop = %+v %v, recorded %+v, drops %v, moves %d, ends %v", out, err, recorded, drops, moves, endIDs)
}
end, _ := ends.said[meshwants.EventStepEnded+".n1"].(meshwants.StepEnded)
if end.ID != "ask-q" || end.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "dropped by hand: stuck") ||
len(end.Shared) != 1 || end.Shared[0].Ask != "run-n2" {
t.Errorf("the reader's end %+v", end)
}
if keys, _ := kv.keys(); len(keys) != 2 || keys[0] != runKeyOf("main.3", "n1") {
t.Errorf("keys after the drop %v, want n1's run and its untaken key", keys)
}
// The tools call the verbs with what they were given.
var asked []string
tools := compositionsTools(func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, key+" "+string(raw))
return json.RawMessage(`{"runs":[],"fill":{"store":"composition-runs","bytes":0,"cap":1}}`), nil
})
for _, tool := range tools {
if _, err := tool.Run(map[string]any{"candidate": "main.3", "machine": "n1", "why": "x"}); err != nil {
t.Fatal(err)
}
}
if len(tools) != 2 || tools[0].Name != compositionsTool || tools[1].Name != compositionsDropTool ||
asked[0] != `seat:mesh-wants.compositions {"candidate":"main.3","machine":"n1"}` || asked[1] != `seat:mesh-wants.compositions-drop {"candidate":"main.3","machine":"n1","why":"x"}` {
t.Errorf("the tools asked %v", asked)
}
}
@@ -23,6 +23,8 @@ type stepOutcome struct {
Outcome string
Said string
Warnings []string
// Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729).
Shared []meshwants.Shared
}
// compositionSteps takes the composition steps and says their ends.
@@ -30,14 +32,21 @@ type compositionSteps struct {
node string
// say says one event of the seat mesh-wants (wantsPublisher).
say publisher
// run runs one step and answers what it came to.
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
now func() time.Time
logf func(string, ...any)
// run runs one step and answers what it came to; resume judges a step taken up at the start from its runs (takeUp).
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
resume func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
// ended writes, once the step asked under id said its end, that time on its reader in each run naming it (A-470m); nil
// writes none.
ended func(id string, run meshwants.StepRun, at time.Time)
now func() time.Time
logf func(string, ...any)
// asker is the name the step's progress names as its asker, read when it is said (stepAskerOf).
asker func() string
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
ticks func(d time.Duration) (<-chan time.Time, func())
mu sync.Mutex
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave).
stops map[string]context.CancelFunc
}
// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its
@@ -69,10 +78,27 @@ func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string {
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
const progressEvery = time.Minute
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context.
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the
// composition runs it waits on (§723, §729).
type stepPhase struct {
mu sync.Mutex
phase string
mu sync.Mutex
phase string
shared []meshwants.Shared
}
// setShared names the runs the step ctx runs waits on, when it runs one.
func setShared(ctx context.Context, shared []meshwants.Shared) {
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
p.mu.Lock()
p.shared = shared
p.mu.Unlock()
}
}
func (p *stepPhase) waitsOn() []meshwants.Shared {
p.mu.Lock()
defer p.mu.Unlock()
return p.shared
}
type phaseKey struct{}
@@ -100,23 +126,97 @@ func (c *compositionSteps) take(ctx context.Context, id string, data []byte) boo
c.logf("a step ask that is no composition step with an id is left: %v", err)
return false
}
go c.runOne(ctx, id, run)
go c.say1(ctx, id, run, c.run, "")
return true
}
// runOne runs one step and says its end.
func (c *compositionSteps) runOne(ctx context.Context, id string, run meshwants.StepRun) {
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the
// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave)
// says nothing: the runner ended it already.
func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
started := c.now().UTC()
ctx, stopWait := context.WithCancel(ctx)
c.mu.Lock()
if c.stops == nil {
c.stops = map[string]context.CancelFunc{}
}
if c.stops[id] != nil { // the step asked under id is running already: it says its own end
c.mu.Unlock()
stopWait()
return
}
c.stops[id] = stopWait
c.mu.Unlock()
defer func() {
c.mu.Lock()
delete(c.stops, id)
c.mu.Unlock()
stopWait()
}()
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
stop := c.sayProgress(id, phase)
o := c.run(context.WithValue(ctx, phaseKey{}, phase), id, run)
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
stop()
if ctx.Err() != nil {
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
return
}
c.sayEnd(id, run, &started, o, again)
}
// sayEnd says the end of the step asked under id as the seat's step-ended.<node>, its id the ask's; once said, its time is
// written as its reader's `ended` (A-470m). Its first end's message id is the ask's id; an end said again is published under
// `<ask id>.<again>`, again the ask of the run whose end makes it say again, so the bus's deduplication never drops it (A-470q).
func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *time.Time, o stepOutcome, again string) {
ended := c.now().UTC()
if started == nil {
started = &ended
}
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module,
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil {
Rerun: run.Rerun, By: run.By, Started: started, Ended: &ended, Shared: o.Shared}
msgID := id
if again != "" {
msgID = id + "." + again
}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil {
c.logf("saying the end of the composition step %s: %v", id, err)
return
}
if c.ended != nil {
c.ended(id, run, ended)
}
}
// move says the end again of each reader a rerun carried into an ended run, marked rerun (§727, A-470d): its machines judged
// from the candidate's keys naming it, the run's new ask in `shared`. A reader running here says its own end.
func (c *compositionSteps) move(ctx context.Context) func(run meshwants.CompositionRun) {
return func(run meshwants.CompositionRun) {
for _, reader := range run.Readers {
if !reader.Rerun || c.running(reader.ID) {
continue
}
c.logf("the composition step %s of %s moves with the composition run of %s (ask %s)", reader.ID, reader.Pipeline, run.Machine, run.Ask)
go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume, run.Ask) // said again under the moved run's ask (A-470q)
}
}
}
// running is whether the step asked under id runs here.
func (c *compositionSteps) running(id string) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.stops[id] != nil
}
// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs).
func (c *compositionSteps) leave(ids []string) {
c.mu.Lock()
defer c.mu.Unlock()
for _, id := range ids {
if stop := c.stops[id]; stop != nil {
stop()
}
}
}
@@ -139,7 +239,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()
return
case <-tick:
at := c.now().UTC()
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at}
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at, Shared: phase.waitsOn()}
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
c.logf("saying the progress of the composition step %s: %v", id, err)
}
@@ -168,7 +168,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
t.Fatal("the composition step said no end")
}
if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") ||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 1 || !strings.Contains(e.Warnings[0], "n3 is away") {
strings.Contains(e.Said, "n1") || len(e.Warnings) != 2 || !strings.Contains(e.Warnings[0], "n3 is away") ||
e.Warnings[1] != "n2: the composer names no culprits" {
t.Fatalf("the step ended %+v", e)
}
if programs["n1"] != engineCommit || programs["n2"] != engineCommit {
@@ -180,9 +181,24 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
t.Error("the machine away was asked for its input")
}
}
keys, _ := b.buckets[buckets.Inputs()].Keys(b.ctx)
if len(keys) != 0 {
t.Errorf("the inputs stay kept past their step-ended: %v", keys)
// The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch.
inputs := b.buckets[buckets.Inputs()]
w, err := inputs.WatchAll(b.ctx)
if err != nil {
t.Fatal(err)
}
defer func() { _ = w.Stop() }()
deadline := time.After(10 * time.Second)
for {
keys, _ := inputs.Keys(b.ctx)
if len(keys) == 0 {
break
}
select {
case <-w.Updates():
case <-deadline:
t.Fatalf("the inputs stay kept past their step-ended: %v", keys)
}
}
}
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"slices"
"strings"
"sync"
"testing"
"time"
@@ -40,7 +42,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
asked = run
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}}
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"},
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
}}
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
if c.take(context.Background(), "ask-0", other) {
@@ -67,7 +70,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil {
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil ||
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
t.Fatalf("step-ended %+v", end)
}
}
@@ -105,6 +109,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
},
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}})
<-release
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
@@ -122,7 +127,8 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
said.mu.Lock()
p, ok := said.said[progress].(meshwants.StepProgressEvent)
said.mu.Unlock()
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) {
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) ||
len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723)
t.Fatalf("step-progress %+v", p)
}
close(release)
@@ -135,3 +141,70 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
t.Fatal("the ticker is not stopped when the step has ended")
}
}
// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end;
// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no
// reader is left", so a later step starts a new run.
func TestASupersededStepLeavesItsRuns(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf)
var drops []string
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
step := func(pipeline string) meshwants.StepRun {
return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3}
}
said := newSaidEvents()
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf,
run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome {
j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id)
if err != nil {
t.Error(err)
}
r.await(ctx, []string{j.key})
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
for _, p := range []string{"main.3.a", "main.3.b"} {
if !c.take(context.Background(), "ask-"+p, body(step(p))) {
t.Fatal("not taken")
}
}
key := runKeyOf("main.3", "n1")
for {
wake := r.wake()
if run, _, _, _ := r.read(key); len(run.Readers) == 2 {
break
}
<-wake
}
ids, err := r.leave("main.3.a", c.leave)
run, _, _, _ := r.read(key)
if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 {
t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops)
}
_, err = r.leave("main.3.b", c.leave)
run, _, _, _ = r.read(key)
if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 ||
!strings.HasPrefix(drops[0], run.Ask+": no reader is left") {
t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops)
}
deadline := time.After(5 * time.Second)
for {
c.mu.Lock()
n := len(c.stops)
c.mu.Unlock()
if n == 0 {
break
}
select {
case <-deadline:
t.Fatal("the steps that left still wait")
case <-time.After(time.Millisecond):
}
}
said.mu.Lock()
defer said.mu.Unlock()
if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok {
t.Errorf("a step that left says its end: %v", said.said)
}
}
@@ -25,14 +25,17 @@ import (
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"mesh-delivery-assigner/internal/buckets"
)
// A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step
// request it asked a build seat to run in its state `inputs`, one key `<time>.<digest>` per input, the value the input's bytes in
// request it asked a build seat to run in its state `inputs`, one key per input, its sha256 alone (A-465a), the value the input's bytes in
// base64 with the state's object store `inputs-objects` holding one over a key's limit (mesh-sdk stateobjects), and the seat
// mesh-wants' verb `input` answers one by its digest in chunks, to the holders of the build seats alone (the verb's callers, §338).
// An input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, whichever is first, the day
// judged at each new ask over at most 20 of the oldest keys. `assigner_inputs` reads them. This mirrors the indexer's
// An input is kept once per content: every ask that names its digest shares it (A-465a). It is deleted, with its object, when the
// step-ended of the last ask that names it is heard, or a day after its last write, whichever is first, the day judged at each new
// ask over at most 20 of the oldest inputs. `assigner_inputs` reads them. This mirrors the indexer's
// (mesh-delivery-indexer internal/index/run_inputs.go): the two are separate modules' code, owed to mesh-sdk.
// inputsTool is the tool that reads the kept inputs.
@@ -45,10 +48,39 @@ const (
inputsSwept = 20
inputsTime = "20060102T150405Z"
// inputsAskPrefix opens the keys that name an input by the id of the ask it was kept for: ask.<id>, the id escaped as a name,
// holding the input's own key, so the build seat's step-ended of that ask finds it.
// holding the digest of the input the ask uses, so the build seat's step-ended of that ask finds it.
inputsAskPrefix = "ask."
)
// inputAsk is what an ask's key holds (A-465b): the digest of the input the ask uses and the time the ask wrote it, in inputsTime.
type inputAsk struct {
Digest string `json:"digest"`
At string `json:"at"`
}
// readAsk reads an ask's key: the JSON form, or the bare key of the old form `<time>.<digest>` an earlier assigner wrote, which
// carries its time in the key itself.
func readAsk(raw []byte) (digest string, at time.Time) {
var a inputAsk
if json.Unmarshal(raw, &a) == nil && a.Digest != "" {
at, _ = time.Parse(inputsTime, a.At)
return a.Digest, at
}
digest = string(raw)
t, _, _ := strings.Cut(digest, ".")
at, _ = time.Parse(inputsTime, t)
return digest, at
}
// isDigest is whether a key is an input's own key: 64 hex digits.
func isDigest(key string) bool {
if len(key) != 64 {
return false
}
_, err := hex.DecodeString(key)
return err == nil
}
// digestOf is an input's digest as the ask names it: its sha256 in hex.
func digestOf(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) }
@@ -64,8 +96,9 @@ type inputKeeper struct {
used func() (int64, error)
}
// keep keeps the exact bytes of the input of the ask under their write time and sha256, after sweeping the keys more than a day old
// among the 20 oldest, and names it by the ask's id; it answers the digest. The same bytes written again in the same second are one key.
// keep keeps the exact bytes of the input of the ask under their sha256, after sweeping, and names the digest and the time in the
// ask's key; it answers the digest. The same bytes asked again are the same one input (A-465a, A-465b). The ask's key is written
// first, so a sweep never finds the input unnamed.
func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
if ask == "" {
return "", errors.New("name the ask the input is kept for")
@@ -73,29 +106,43 @@ func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
if err := k.sweep(); err != nil {
k.logf("inputs: the keys older than a day are not all deleted: %v", err)
}
if k.used != nil {
used, err := k.used()
if err != nil {
return "", fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", meshwants.StateInputs, err)
}
if used+int64(len(body)) > inputsCap {
return "", fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
meshwants.StateInputs, inputsCap>>20, used, len(body))
}
}
digest := digestOf(body)
key := meshwants.InputsByTimeDigestKey(k.now().UTC().Format(inputsTime), digest)
if err := k.vals.Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
raw, err := json.Marshal(inputAsk{Digest: digest, At: k.now().UTC().Format(inputsTime)})
if err != nil {
return "", err
}
if err := k.vals.KV.Create(inputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
askKey := inputsAskPrefix + keys.EscapeName(ask)
if err := k.vals.KV.Create(askKey, raw); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
have, err := k.vals.KV.Get(digest)
if err != nil {
return "", err
}
if have != nil {
return digest, nil
}
if k.used != nil {
used, err := k.used()
if err == nil && used+int64(len(body)) > inputsCap {
err = fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
buckets.InputsState, inputsCap>>20, used, len(body))
} else if err != nil {
err = fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", buckets.InputsState, err)
}
if err != nil {
_ = k.vals.KV.Delete(askKey)
return "", err
}
}
if err := k.vals.Put(digest, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
return digest, nil
}
// ended deletes the input kept for an ask when the build seat's step-ended of that ask is heard; an ask that kept none, or whose
// input a sweep deleted, changes nothing.
// ended removes the ask's key when the build seat's step-ended of that ask is heard, and deletes the input it named, with its
// object, when no other ask's key names it (A-465a); an ask that kept none changes nothing.
func (k *inputKeeper) ended(ask string) error {
if ask == "" {
return nil
@@ -105,10 +152,41 @@ func (k *inputKeeper) ended(ask string) error {
if err != nil || e == nil {
return err
}
if err := k.deleteInput(string(e.Value)); err != nil {
digest, _ := readAsk(e.Value)
if err := k.vals.KV.Delete(askKey); err != nil {
return err
}
return k.vals.KV.Delete(askKey)
named, err := k.named()
if err != nil {
return err
}
if named[digest] {
return nil // another running ask still names it
}
return k.deleteInput(digest)
}
// named is the inputs some ask's key names now.
func (k *inputKeeper) named() (map[string]bool, error) {
all, err := k.vals.KV.Keys()
if err != nil {
return nil, err
}
named := map[string]bool{}
for _, key := range all {
if !strings.HasPrefix(key, inputsAskPrefix) {
continue
}
o, err := k.vals.KV.Get(key)
if err != nil {
return nil, err
}
if o != nil {
d, _ := readAsk(o.Value)
named[d] = true
}
}
return named, nil
}
// deleteInput deletes an input's key and the object that key names, when it names one (A-465): an owner that ends its own key
@@ -163,60 +241,76 @@ func (k *inputKeeper) dropAtStart() {
k.logf("inputs: %d objects of deleted keys removed at the start, %d bytes", len(dropped), bytes)
}
// sweep deletes, among the 20 oldest keys, those written more than a day ago, and a name by ask whose input is gone.
// sweep deletes, among the 20 oldest ask keys, those written more than a day ago; every input of the digest form no ask's key names
// now, with its object; a key of the old form `<time>.<digest>` a day after the time it carries; and an old-form ask key whose
// input is gone (A-465b).
func (k *inputKeeper) sweep() error {
all, err := k.vals.KV.Keys()
if err != nil {
return err
}
var errs []error
var inputs, asks []string
type askAt struct {
key string
at time.Time
}
var asks []askAt
var inputs []string
for _, key := range all {
if strings.HasPrefix(key, inputsAskPrefix) {
asks = append(asks, key)
} else {
inputs = append(inputs, key)
}
}
slices.Sort(inputs) // the write time leads the key, so the oldest come first
if len(inputs) > inputsSwept {
inputs = inputs[:inputsSwept]
}
for _, key := range inputs {
t, _, ok := meshwants.ParseInputsByTimeDigestKey(key)
at, perr := time.Parse(inputsTime, t)
if !ok || perr != nil || k.now().Sub(at) < inputsKeptFor {
continue
}
if err := k.deleteInput(key); err != nil {
errs = append(errs, err)
}
}
for _, key := range asks {
if e, err := k.vals.KV.Get(key); err == nil && e != nil {
if gone, err := k.vals.KV.Get(string(e.Value)); err == nil && gone == nil {
if err := k.vals.KV.Delete(key); err != nil {
errs = append(errs, err)
e, err := k.vals.KV.Get(key)
if err != nil || e == nil {
continue
}
d, at := readAsk(e.Value)
if !isDigest(d) {
if gone, err := k.vals.KV.Get(d); err == nil && gone == nil {
errs = append(errs, k.vals.KV.Delete(key))
}
}
asks = append(asks, askAt{key, at})
continue
}
if !isDigest(key) {
t, _, _ := strings.Cut(key, ".")
if at, err := time.Parse(inputsTime, t); err == nil && k.now().Sub(at) >= inputsKeptFor {
errs = append(errs, k.deleteInput(key))
}
continue
}
inputs = append(inputs, key)
}
slices.SortFunc(asks, func(a, b askAt) int { return a.at.Compare(b.at) })
if len(asks) > inputsSwept {
asks = asks[:inputsSwept]
}
for _, a := range asks {
if !a.at.IsZero() && k.now().Sub(a.at) >= inputsKeptFor {
errs = append(errs, k.vals.KV.Delete(a.key))
}
}
named, err := k.named()
if err != nil {
return errors.Join(append(errs, err)...)
}
for _, key := range inputs {
if !named[key] {
errs = append(errs, k.deleteInput(key))
}
}
return errors.Join(errs...)
}
// keyOf is the key of the input of the digest, or none.
// keyOf is the key of the input of the digest, or none: the digest itself.
func (k *inputKeeper) keyOf(digest string) (string, error) {
all, err := k.vals.KV.Keys()
if err != nil {
if !isDigest(digest) {
return "", nil
}
e, err := k.vals.KV.Get(digest)
if err != nil || e == nil {
return "", err
}
slices.Sort(all)
for _, key := range all {
if _, d, ok := meshwants.ParseInputsByTimeDigestKey(key); ok && d == digest && !strings.HasPrefix(key, inputsAskPrefix) {
return key, nil
}
}
return "", nil
return digest, nil
}
// body is the exact bytes of the input of the digest, checked against it, whatever its size.
@@ -226,7 +320,7 @@ func (k *inputKeeper) body(digest string) ([]byte, error) {
}
key, err := k.keyOf(digest)
if err != nil {
return nil, fmt.Errorf("the state %s cannot be read: %w", meshwants.StateInputs, err)
return nil, fmt.Errorf("the state %s cannot be read: %w", buckets.InputsState, err)
}
if key == "" {
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
@@ -294,7 +388,7 @@ func (o *inputObjects) open() (stateobjects.Objects, error) {
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(meshwants.StateInputs)))
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(buckets.InputsState)))
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
return nil, fmt.Errorf("the object store of the state inputs is not on the bus yet: %w", process.ErrNotYet)
}
@@ -345,7 +439,7 @@ func inputsOver(kv jetstream.KeyValue, js jetstream.JetStream, module string, lo
return nil
}
objects := &inputObjects{js: js, module: module}
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(meshwants.StateInputs)}
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(buckets.InputsState)}
used := func() (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
@@ -386,12 +480,21 @@ func inputsRead(k *inputKeeper) stdio.Tool {
return nil, fmt.Errorf("the inputs cannot be listed: %w", err)
}
slices.Sort(all)
lastAsk := map[string]string{} // the latest write time an ask's key names for each input
for _, key := range all {
if e, err := k.vals.KV.Get(key); err == nil && e != nil && strings.HasPrefix(key, inputsAskPrefix) {
if d, at := readAsk(e.Value); !at.IsZero() && at.Format(inputsTime) > lastAsk[d] {
lastAsk[d] = at.Format(inputsTime)
}
}
}
out := []inputSaid{}
for _, key := range all {
t, d, ok := meshwants.ParseInputsByTimeDigestKey(key)
if !ok || strings.HasPrefix(key, inputsAskPrefix) || (want != "" && d != want) {
d := key
if !isDigest(key) || (want != "" && d != want) {
continue
}
t := lastAsk[key]
var text string
if found, err := k.vals.Get(key, &text); err != nil || !found {
continue
@@ -451,22 +554,28 @@ func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error))
}}
}
// heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the
// composition run that waits for that ask (A-240); one whose input cannot be deleted is given again by the bus.
func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(string, ...any)) {
// heardStepEnded hands the end a build seat's step-ended names to the composition run of that ask (A-240, §719), which writes it
// in the run's key, and then deletes the input kept for the ask (design 47 G §354): the run's end first, so no other act delays
// it past its bound (A-470o, A-470p). One whose run's end cannot be written, or whose input cannot be deleted, is given again by
// the bus; its redelivery finds the run ended, said as ignored for the run, and still deletes the input.
func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) {
var e buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if runEnded != nil {
if err := runEnded(e); err != nil {
logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err)
_ = m.Nak()
return
}
}
if err := k.ended(e.Ask); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
_ = m.Nak()
return
}
if ends != nil {
ends.heard(e)
}
_ = m.Ack()
}
@@ -3,6 +3,7 @@ package main
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
@@ -15,6 +16,7 @@ import (
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
@@ -23,6 +25,8 @@ type kvMem struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
// delErr, when set, fails the next Delete once.
delErr error
}
func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} }
@@ -57,6 +61,10 @@ func (k *kvMem) Update(key string, v []byte, rev uint64) error {
func (k *kvMem) Delete(key string) error {
k.mu.Lock()
defer k.mu.Unlock()
if err := k.delErr; err != nil {
k.delErr = nil
return err
}
delete(k.vals, key)
return nil
}
@@ -246,6 +254,47 @@ func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) {
if _, err := k.body(d2); err == nil {
t.Error("an input a day old is still kept at the next ask")
}
// A-470o, A-470p: the step-ended's run end is written before the input is deleted; a delete that fails asks the event again,
// whose redelivery finds the run ended and still deletes the input; a run end not written asks it again with the input kept.
for _, c := range []struct {
name string
delErr error
runErr error
naked bool
runs int
deleted bool
}{
{name: "the run's end before the input's delete", runs: 1, deleted: true},
{name: "an input not deleted asks the event again", delErr: errors.New("the bus is away"), naked: true, runs: 2, deleted: true},
{name: "a run's end not written asks the event again", runErr: errors.New("the run is not read"), naked: true, runs: 1},
} {
t.Run(c.name, func(t *testing.T) {
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
d, _ := k.keep("ask/run", []byte(`{"heads":"abc"}`))
k.vals.KV.(*kvMem).delErr = c.delErr
runs := 0
runEnded := func(e buildstep.StepEnded) error {
runs++
if _, err := k.body(d); err != nil {
t.Errorf("the run's end is written after the input's delete (%d)", runs)
}
return c.runErr
}
raw, _ := json.Marshal(buildstep.StepEnded{Ask: "ask/run"})
m := &heardMsg{data: raw, subject: "step-ended"}
heardStepEnded(k, runEnded, m, t.Logf)
if m.naked != c.naked || m.acked == c.naked {
t.Fatalf("acked %v, naked %v, want naked %v", m.acked, m.naked, c.naked)
}
if m.naked && c.delErr != nil { // the redelivery
m = &heardMsg{data: raw, subject: "step-ended"}
heardStepEnded(k, func(e buildstep.StepEnded) error { runs++; return nil }, m, t.Logf)
}
if _, err := k.body(d); runs != c.runs || (err != nil) != c.deleted {
t.Errorf("the run's end heard %d times (want %d), the input deleted %v (want %v)", runs, c.runs, err != nil, c.deleted)
}
})
}
}
// Design 47 G A-465: an input the assigner ends itself (the step-ended of its ask, the sweep of a day-old key) ends its object
@@ -346,3 +395,109 @@ func TestAssignerInputsSaysWhenTheKeyGoneObjectsCannotBeListed(t *testing.T) {
t.Errorf("answer %+v: an unreadable store is not answered as no key-gone objects", m)
}
}
// Design 47 G A-465a: an input is kept once per content under its sha256 alone; every ask names the digest it uses, and the input
// goes, with its object, at the step-ended of the last ask that names it, or a day after its last write; a request the cap still
// refuses says the fill; an input of the old form (a key that is no digest) is ended with its object.
func TestAnInputIsKeptOncePerContentAndEndsWithItsLastAsk(t *testing.T) {
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, func() time.Time { return now })
body := bytes.Repeat([]byte{'n'}, 2<<20)
var digests []string
for _, ask := range []string{"ask-1", "ask-2", "ask-3"} {
d, err := k.keep(ask, body)
if err != nil {
t.Fatal(err)
}
digests = append(digests, d)
}
if digests[0] != digestOf(body) || digests[1] != digests[0] || digests[2] != digests[0] {
t.Fatalf("digests = %v, want the sha256 of the bytes each time", digests)
}
if _, err := k.vals.KV.Get(digests[0]); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(digests[0]); e == nil || len(objs.objs) != 1 {
t.Fatalf("one key by the digest and one object wanted, got entry %v and %d objects", e, len(objs.objs))
}
for _, key := range keysOf(t, k) {
if !strings.HasPrefix(key, inputsAskPrefix) && key != digests[0] {
t.Errorf("a key %q beside the one input", key)
}
}
if e, _ := k.vals.KV.Get(inputsAskPrefix + "ask-2"); e == nil || !strings.Contains(string(e.Value), digests[0]) || !strings.Contains(string(e.Value), "20261117T100000Z") {
t.Errorf("the ask does not name the digest it uses: %v", e)
}
for _, ask := range []string{"ask-1", "ask-2"} {
if err := k.ended(ask); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Fatalf("the input went at the end of %s while ask-3 still names it: %v", ask, err)
}
}
if err := k.ended("ask-3"); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 || len(keysOf(t, k)) != 0 {
t.Errorf("the last step-ended left the input (%v), %d objects, keys %v", err, len(objs.objs), keysOf(t, k))
}
// a day after its last write, a renewed write counted
if _, err := k.keep("ask-4", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-5", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-6", []byte("other")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Errorf("an input written again 20 hours ago was swept: %v", err)
}
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask-7", []byte("another")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 {
t.Errorf("an input a day after its last write is kept (%v), %d objects", err, len(objs.objs))
}
// an input of the old form goes a day after the time its key carries, with its object, never earlier
old := "20261117T090000Z." + digestOf([]byte("old"))
if err := k.vals.Put(old, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{'o'}, 2<<20))); err != nil {
t.Fatal(err)
}
recent := now.Add(-time.Hour).Format(inputsTime) + "." + digestOf([]byte("recent"))
if err := k.vals.Put(recent, "cmVjZW50"); err != nil {
t.Fatal(err)
}
if err := k.sweep(); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(old); e != nil || len(objs.objs) != 0 {
t.Errorf("the input of the old form is kept (%v) with %d objects", e, len(objs.objs))
}
if e, _ := k.vals.KV.Get(recent); e == nil {
t.Error("an input of the old form under a day old was deleted early")
}
// a new content the cap refuses says the fill; the same content, already kept, needs no room
full := newTestKeeperWithFill(objs, now, inputsCap-10)
if _, err := full.keep("ask-8", []byte("fits")); err != nil {
t.Fatalf("a content that fits is refused: %v", err)
}
if _, err := full.keep("ask-9", []byte("does not fit at all")); err == nil || !strings.Contains(err.Error(), "bytes kept") {
t.Errorf("the refusal does not name the fill: %v", err)
}
if _, err := full.keep("ask-10", []byte("fits")); err != nil {
t.Errorf("the content already kept is refused at the cap: %v", err)
}
}
func newTestKeeperWithFill(objs *objectsMem, now time.Time, used int64) *inputKeeper {
k := testKeeper(objs, func() time.Time { return now })
k.used = func() (int64, error) { return used, nil }
return k
}
@@ -69,11 +69,13 @@ import (
type bed struct {
// seatListRead is closed once the assigner read the seat list (serveSeatList).
seatListRead <-chan struct{}
srv *server.Server
js jetstream.JetStream
ctx context.Context
dir string
buckets map[string]jetstream.KeyValue
// settings are the files of settings a test sets, by the variable naming each (as the manifest's resources fill them).
settings map[string]string
srv *server.Server
js jetstream.JetStream
ctx context.Context
dir string
buckets map[string]jetstream.KeyValue
// placed is, by node, the modules the bed's assignments names on it (place).
placed map[string][]string
// journal is what the programs the bed starts write in their journal.
@@ -169,7 +171,7 @@ func startBed(t *testing.T, leaveOut string) *bed {
continue
}
cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
if name == plc.PulsesBucket() {
if name == plc.PulsesBucket() || name == buckets.CompositionRuns() {
cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
}
kv, err := js.CreateKeyValue(ctx, cfg)
@@ -301,6 +303,13 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": settle}
for name, value := range b.settings {
file := filepath.Join(b.dir, name)
if err := os.WriteFile(file, []byte(value), 0o600); err != nil {
t.Fatal(err)
}
env[name] = file
}
ctx, stop := context.WithCancel(b.ctx)
done := make(chan error, 1)
go func() {
@@ -9,8 +9,10 @@ package main
import (
"context"
"fmt"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"io"
"log"
@@ -89,7 +91,7 @@ func ownBuckets() []string {
return []string{buckets.Asks(), buckets.Differences(),
buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(),
plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(),
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket()}
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), buckets.CompositionRuns(), pinsBucket()}
}
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
@@ -133,11 +135,12 @@ var keptStates = []announce.State{
{Name: copiesState, Readable: true},
{Name: writtenState, Readable: true},
{Name: buckets.WaitsState, Readable: true},
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c)
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
}
// kvSet is every bucket the assigner opened, by name.
@@ -526,9 +529,34 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// The build seat's step-ended of an ask deletes the input kept for it (design 47 G §354) and ends the composition run that waits
// for it (A-240); the watchman's machine-away and machine-back say which machines a composition step does not compose (§64,
// §389). Registered before the hub starts.
ends, away := &runEnds{}, &awayNodes{}
away := &awayNodes{}
// The composition runs (design 47 G §718, §719), watched in their own state; a build seat's step-ended of a run's ask ends
// the run in its key.
var runs *compositionRuns
if kv := kvs[buckets.CompositionRuns()]; kv != nil {
set, err := runSettingsOf(getenv)
if err != nil {
return err
}
runs = newCompositionRuns(busRuns{ctx, js, kv}, set, time.Now, logger.Printf)
if js != nil {
runs.say = wantsPublisher(ctx, js, thisNode)
}
if bus != nil {
runs.drop = dropOver(bus.CallKept)
}
// The build seats' ask-taken and step-progress begin a run's clock (§720, A-466q).
if err := hearRunBounds(hub, runs, logger.Printf); err != nil {
logger.Printf("the build seats' ask-taken and step-progress are not heard, so a composition run is ended by no bound: %v", err)
}
}
if inputs != nil {
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, ends, m, logger.Printf) }); err != nil {
var runEnded func(buildstep.StepEnded) error
if runs != nil && bus != nil {
read := func(e buildstep.StepEnded) ([]byte, error) { return e.ReadOutput(buildstep.AskGo, bus.CallFile) }
runEnded = func(e buildstep.StepEnded) error { return runs.heardEnd(e, read) }
}
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, runEnded, m, logger.Printf) }); err != nil {
return err
}
}
@@ -682,11 +710,40 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
p.acts = newPutActs(bus, wake, logger.Printf)
}
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, ended: runs.stepEnded, now: time.Now, logf: logger.Printf,
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
move := steps.move(ctx) // the readers a rerun carried (§727, A-470d)
runs.moved.Store(&move)
// The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up
// once the values that stand are read (§719, §720).
if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, runs.expired, func() { steps.takeUp(ctx, runs) }); err != nil {
return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err)
}
// The runs' read and drop, the seat mesh-wants' verbs compositions and compositions-drop (§728, A-469f), answered here
// where the runs are kept and the readers' ends said; the drop's hand act recorded through the seat mesh-hand-acts first.
if e, ok := bus.(emitting); ok {
record := func(q meshhandactsclient.Request) error {
_, err := meshhandactsclient.Recorder{Ask: bus.Call}.Record(q)
return err
}
stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), compositionVerbs(runs, steps, record))
if err != nil {
logger.Printf("the verbs %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, err)
} else {
go func() { <-ctx.Done(); stopVerbs() }()
}
}
// A superseded or cancelled pipeline's composition step leaves the runs it reads (§723, A-469n).
if p.ends != nil {
left := func(pipeline string) error {
_, err := runs.leave(pipeline, steps.leave)
return err
}
p.ends.left.Store(&left)
}
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
} else {
@@ -45,7 +45,9 @@ type passer struct {
kvs kvSet
w *world
logf func(string, ...any)
sdk string
// unread is the set of modules whose running build's manifest served last could not read, said once per change of it.
unread saidSet
sdk string
// say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a
// pass says a condition once and a restart says it again (the router keeps one per key).
say func(oc.Condition) error
@@ -1434,6 +1434,17 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
t.Errorf("served(%s) = %v, want %v: the running server's manifest serves w alone, late's is not read", v, served(v), want)
}
}
// The modules whose running build's manifest is not read are said once per change of their set (G-23).
var said []string
p.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
lateOnly := assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"server": at(fp("f")), "late": at(fp("2"))}}}
p.served(g, lateOnly)
p.served(g, lateOnly)
p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"late": at(fp("2")), "loop": at(fp("1"))}}})
p.served(g, lateOnly)
if len(said) != 2 || !strings.Contains(said[0], "[loop late]") && !strings.Contains(said[0], "[late loop]") || !strings.Contains(said[1], "[late]") {
t.Errorf("the unread manifests were said %q, want once per change of the set", said)
}
// A-426a: a healthy build held (no catalog entry of it) or waiting for a yes serves what it serves, said once; one held unhealthy,
// or whose running statement says it stopped, does not.
held := at(fp("2"))
@@ -153,6 +153,9 @@ type pipelineEnds struct {
firsts map[string]string
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
runEnded func(group string) error
// left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read
// (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts.
left atomic.Pointer[func(pipeline string) error]
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
onBranches func([]string)
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
@@ -285,6 +288,13 @@ func (e *pipelineEnds) branchOf(module, placed string) string {
// §29, §64).
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
p, err := mprc.PipelineEnded(m.Data())
if left := e.left.Load(); err == nil && left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
if err := (*left)(p.Pipeline); err != nil {
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
_ = m.Nak()
return
}
}
switch {
case err != nil:
e.logf("%s is not taken: %v", m.Subject(), err)
@@ -423,6 +423,7 @@ func serveTools() error {
tools = append(tools, wants.tools()...)
keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}
tools = append(tools, inputsRead(keeper), inputsDrop(keeper, stdio.Ask))
tools = append(tools, compositionsTools(stdio.Ask)...) // the composition runs' read and drop, through their verbs (§728)
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
@@ -2,6 +2,8 @@ package main
import (
"slices"
"strings"
"sync"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
@@ -53,12 +55,31 @@ func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
}
served, unread := graph.Served(servingHaves(n, p.countedServing), func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
func(module string) []string { return g[module].Serves })
if len(unread) > 0 {
if p.unread.changed(unread) && len(unread) > 0 {
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
}
return served
}
// saidSet is the last set of names said, so a pass says a set again only when it changed.
type saidSet struct {
mu sync.Mutex
last string
}
// changed keeps names as the set said last and says whether it differs from the one before.
func (s *saidSet) changed(names []string) bool {
sorted := slices.Sorted(slices.Values(names))
key := strings.Join(sorted, " ")
s.mu.Lock()
defer s.mu.Unlock()
if key == s.last {
return false
}
s.last = key
return true
}
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A have
// waiting for a person's yes on a newer build counts too: it names the build that still runs and that build's health, and
@@ -29,6 +29,9 @@ const (
PutWaitsState = "put-waits"
// InputsState holds the inputs of the composition steps, `<time>.<digest>`, which the seat mesh-wants' verb input answers (design 47 G §354).
InputsState = "inputs"
// CompositionRunsState holds the composition runs, one key `<candidate key>.<machine>` per run and its bound keys
// `bound.<run key>` and `untaken.<run key>` (design 47 G §719, §720, A-470).
CompositionRunsState = "composition-runs"
// PoliciesState is the state of the policies a person set for a module's upgrade.
PoliciesState = "policies"
)
@@ -37,16 +40,17 @@ const (
// constant of its name (design 47 G A-296, §428).
func own(state string) string { return plc.Module() + "_" + state }
func Wants() string { return own(wants.StateWants) }
func Differences() string { return own(wants.StateDifferences) }
func Asks() string { return own("asks") }
func Puts() string { return own("puts") }
func Waits() string { return own(WaitsState) }
func DeliveredAgain() string { return own(DeliveredAgainState) }
func DeliveryWaits() string { return own(DeliveryWaitsState) }
func PutWaits() string { return own(PutWaitsState) }
func Inputs() string { return own(InputsState) }
func Policies() string { return own(PoliciesState) }
func Wants() string { return own(wants.StateWants) }
func Differences() string { return own(wants.StateDifferences) }
func Asks() string { return own("asks") }
func Puts() string { return own("puts") }
func Waits() string { return own(WaitsState) }
func DeliveredAgain() string { return own(DeliveredAgainState) }
func DeliveryWaits() string { return own(DeliveryWaitsState) }
func PutWaits() string { return own(PutWaitsState) }
func Inputs() string { return own(InputsState) }
func CompositionRuns() string { return own(CompositionRunsState) }
func Policies() string { return own(PoliciesState) }
// AsksHistory is the revisions the bucket of asks keeps per key.
const AsksHistory = 5
@@ -31,7 +31,8 @@ func TestTheAssignersOwnBucketsTakeTheModuleItRunsAs(t *testing.T) {
"Wants": {Wants(), "some-assigner_wants"}, "Differences": {Differences(), "some-assigner_differences"},
"Asks": {Asks(), "some-assigner_asks"}, "Puts": {Puts(), "some-assigner_puts"}, "Policies": {Policies(), "some-assigner_policies"},
"Waits": {Waits(), "some-assigner_waits"}, "Inputs": {Inputs(), "some-assigner_inputs"},
"Assignments": {plc.AssignmentsBucket(), "some-assigner_assignments"}, "SeatHolders": {plc.SeatHoldersBucket(), "some-assigner_seat-holders"},
"CompositionRuns": {CompositionRuns(), "some-assigner_composition-runs"},
"Assignments": {plc.AssignmentsBucket(), "some-assigner_assignments"}, "SeatHolders": {plc.SeatHoldersBucket(), "some-assigner_seat-holders"},
} {
if c[0] != c[1] {
t.Errorf("%s = %q, want %q", name, c[0], c[1])
@@ -70,7 +70,7 @@ func TestManifest(t *testing.T) {
}
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
"assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1,
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "delivery-waits": 1, "delivered-again": 1, "inputs": 1, "pins": 5}; !reflect.DeepEqual(states, want) {
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "delivery-waits": 1, "delivered-again": 1, "inputs": 1, "composition-runs": 1, "pins": 5}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -93,7 +93,7 @@ func TestManifest(t *testing.T) {
// revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB.
if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh ||
!slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) ||
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input"}) {
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop"}) {
t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims)
}
if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 {
@@ -103,7 +103,7 @@ func TestManifest(t *testing.T) {
// ADR 0621 §1: the first-view tools assignments, seat-holders, wants and asks (§717) and design 25 § 12.2's six read tools, of the
// program's bundle, which runs in a runtime of its own on the assigner's account; ADR 0618 §3: and assigner_engine_reads,
// which lists and asks again what each node's engine reads.
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assignments", "seat-holders", "wants", "asks",
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks",
"assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies",
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_pins", "assigner_unpin", "assigner_delivered_again"}) {
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools)
@@ -137,7 +137,7 @@ func TestManifest(t *testing.T) {
// ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events;
// issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears; §376, §389: its own version-delivered and
// the watchman's machine-away and machine-back, read whole at its start.
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-wants.version-delivered", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.IndexerModule + ".take-in.taken", buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) {
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-wants.version-delivered", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.IndexerModule + ".take-in.taken", buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-build-go.ask-taken", "node-build-go.step-progress", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) {
t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes)
}
if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
@@ -147,7 +147,8 @@ func TestManifest(t *testing.T) {
"seat:mesh-delivery.source", // §507: the branches the gate keeps a source revision of, before the first registrations
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
"seat:node-backup.backed-up", "seat:node-backup.now",
"seat:node-build-go.accept.check", // A-231: the composition run's ask on the Go build seat's queue check (accept.check.<asker>)
"seat:node-build-go.accept.check", // A-231
"seat:node-build-go.drop", // §720, A-469s: a composition run past its bound drops its ask: the composition run's ask on the Go build seat's queue check (accept.check.<asker>)
"seat:node-build-go.output", // A-254: a composition run's answer kept as the build seat's object
"seat:node-engine.compose-candidate", // §354, A-226: each machine's composition input
"seat:node-engine.manifest-reads", // ADR 0618 §3
@@ -167,13 +168,21 @@ func TestManifest(t *testing.T) {
}
file, _ := os.ReadFile("module.json")
if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) ||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 3 {
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 8 ||
raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" ||
raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) {
t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err)
}
for _, f := range []struct{ env, path, key string }{
{"MESH_ASSIGNER_OUTSIDE_WAIT_FILE", "${dir:state}/outside-wait-minutes", "outside-wait-minutes"},
{"MESH_ASSIGNER_OUTSIDE_SETTLE_FILE", "${dir:state}/outside-settle-seconds", "outside-settle-seconds"},
{"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE", "${dir:state}/ask-operator-for", "ask-operator-for"},
// design 47 G §719, §720, §721, A-469x: the composition runs' settings
{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE", "${dir:state}/composition-runs-cap-mib", "composition-runs-cap-mib"},
{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE", "${dir:state}/composition-runs-kept", "composition-runs-kept"},
{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE", "${dir:state}/composition-step-bound", "composition-step-bound"},
{"MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE", "${dir:state}/run-untaken-bound", "run-untaken-bound"},
{"MESH_ASSIGNER_BLAME_TRIES_FILE", "${dir:state}/blame-tries", "blame-tries"},
} {
if !strings.Contains(string(file), `"`+f.env+`": "`+f.path+`"`) || !strings.Contains(string(file), `"path": "`+f.path+`"`) ||
!strings.Contains(string(file), `"content": "${config:`+f.key+`}"`) {
+89 -2
View File
@@ -25,7 +25,9 @@
"revision",
"entry",
"difference",
"input"
"input",
"compositions",
"compositions-drop"
]
}
],
@@ -59,6 +61,8 @@
"mesh-pipeline-runner.pipeline-ended",
"mesh-pipeline-runner.group-ended",
"node-build-go.step-ended",
"node-build-go.ask-taken",
"node-build-go.step-progress",
"node-engine.manifest-reads-changed",
"node-engine.running",
"mesh-delivery.pinned",
@@ -82,6 +86,7 @@
"seat:node-backup.backed-up",
"seat:node-backup.now",
"seat:node-build-go.output",
"seat:node-build-go.drop",
"seat:node-engine.compose-candidate",
"seat:node-engine.manifest-reads",
"seat:operator-channel.asks",
@@ -113,6 +118,8 @@
"assigner_waits",
"assigner_inputs",
"assigner_inputs_drop",
"assigner_compositions",
"assigner_compositions_drop",
"assignments",
"seat-holders",
"wants",
@@ -158,6 +165,7 @@
{ "name": "delivery-waits", "history": 1 },
{ "name": "delivered-again", "history": 1 },
{ "name": "inputs", "history": 1, "max-mib": 64 },
{ "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 },
{ "name": "pins", "history": 5, "max-mib": 4 }
],
"config": {
@@ -182,6 +190,40 @@
"description": "the change kinds of a proposal the assigner asks the operator before it writes it, separated by commas: place, take-off, pin, unpin, hold, unhold, decide, config, add-node, remove-node, and stops for a verdict whose answers say a node would stop running a module (design 47 G A-425, A-425a). A proposal none of whose changes is of a kind named is applied at once when its verdict passes, said in the log and by assignments-changed. The default is take-off,remove-node,stops; empty asks for nothing.",
"trusted": false,
"default": "take-off,remove-node,stops"
},
"composition-runs-cap-mib": {
"type": "number",
"description": "the cap of the state composition-runs in MiB, the process refusing a new composition run at it with its steps ending could not run naming the fill (design 47 G section 719, A-469d); its maximum is the manifest's max-mib.",
"minimum": 1,
"maximum": 64,
"trusted": false,
"default": 16
},
"composition-runs-kept": {
"type": "string",
"description": "how long a composition run's key is kept after it ended, a Go duration such as 24h (design 47 G section 719, A-469x).",
"trusted": false,
"default": "24h"
},
"composition-step-bound": {
"type": "string",
"description": "how long a taken composition run's ask may go without saying its progress before the run ends could not run, a Go duration such as 30m; written again at each step-progress (design 47 G section 720, A-469x).",
"trusted": false,
"default": "30m"
},
"run-untaken-bound": {
"type": "string",
"description": "how long a composition run's ask may wait for a build node to take it before the run ends could not run, a Go duration such as 24h (design 47 G section 720, A-466s, A-469x).",
"trusted": false,
"default": "24h"
},
"blame-tries": {
"type": "number",
"description": "the most compositions a machine's composer makes to find the modules it blames, carried in the run's request as blame_tries (design 47 G section 721, A-466r).",
"minimum": 1,
"maximum": 1000,
"trusted": false,
"default": 20
}
},
"objects": [
@@ -231,6 +273,46 @@
"mode": "0600",
"content": "${config:ask-operator-for}"
},
{
"id": "composition-runs-cap-mib",
"type": "file",
"path": "${dir:state}/composition-runs-cap-mib",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:composition-runs-cap-mib}"
},
{
"id": "composition-runs-kept",
"type": "file",
"path": "${dir:state}/composition-runs-kept",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:composition-runs-kept}"
},
{
"id": "composition-step-bound",
"type": "file",
"path": "${dir:state}/composition-step-bound",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:composition-step-bound}"
},
{
"id": "run-untaken-bound",
"type": "file",
"path": "${dir:state}/run-untaken-bound",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:run-untaken-bound}"
},
{
"id": "blame-tries",
"type": "file",
"path": "${dir:state}/blame-tries",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:blame-tries}"
},
{
"id": "assigner",
"type": "process",
@@ -243,7 +325,12 @@
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_ASSIGNER_OUTSIDE_WAIT_FILE": "${dir:state}/outside-wait-minutes",
"MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": "${dir:state}/outside-settle-seconds",
"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE": "${dir:state}/ask-operator-for"
"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE": "${dir:state}/ask-operator-for",
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "${dir:state}/composition-runs-cap-mib",
"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "${dir:state}/composition-runs-kept",
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "${dir:state}/composition-step-bound",
"MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound",
"MESH_ASSIGNER_BLAME_TRIES_FILE": "${dir:state}/blame-tries"
},
"user": "mesh-delivery-assigner",
"health": {
@@ -15,18 +15,20 @@ import (
const Seat = "mesh-pipeline-runner"
const (
VerbRun = "run"
VerbStart = "start"
VerbShow = "show"
VerbList = "list"
VerbStands = "stands"
VerbWhyCarried = "why-carried"
VerbKept = "kept"
VerbReview = "review"
VerbRerun = "rerun"
VerbAccept = "accept"
VerbCancel = "cancel"
VerbAskAgain = "ask-again"
VerbRun = "run"
VerbStart = "start"
VerbShow = "show"
VerbList = "list"
VerbStands = "stands"
VerbWhyCarried = "why-carried"
VerbKept = "kept"
VerbReview = "review"
VerbRerun = "rerun"
VerbAccept = "accept"
VerbCancel = "cancel"
VerbAskAgain = "ask-again"
VerbTestRuns = "test-runs"
VerbTestRunsDrop = "test-runs-drop"
)
const (
@@ -45,7 +47,7 @@ const (
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
func Verbs() []string {
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain}
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain, VerbTestRuns, VerbTestRunsDrop}
}
// Events is every event published on the seat's subject: a new copy each call.
@@ -596,6 +598,26 @@ type RunStarted struct {
V int64 `json:"v"`
}
// SharedAskedBy is the step that asked the run, for a test step reading another's run.
type SharedAskedBy struct {
// its artifact, <module>:<artifact>
Artifact string `json:"artifact"`
// its pipeline's key
Pipeline string `json:"pipeline"`
}
// Shared is a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself.
type Shared struct {
// the id of the run's ask of the build seat
Ask string `json:"ask"`
// the step that asked the run, for a test step reading another's run
AskedBy *SharedAskedBy `json:"asked-by,omitempty"`
// the machine composed, for a composition run
Machine string `json:"machine,omitempty"`
// the run's key: in the assigner's composition-runs, or the runner's test-runs
Run string `json:"run"`
}
// StepEvent is a step's start or end: the step, its status, its outcome, its fingerprint, its test counts, what a carried result was carried from, and for a rerun that it is one and its proved caller; a bypassed step too.
type StepEvent struct {
At *time.Time `json:"at,omitempty"`
@@ -645,7 +667,9 @@ type StepResult struct {
// the step was asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the seat that serves the step, whose holder alone may end it (ADR 0666 section 21)
Seat string `json:"seat,omitempty"`
Seat string `json:"seat,omitempty"`
// the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t)
Shared []Shared `json:"shared,omitempty"`
Started *time.Time `json:"started,omitempty"`
// the step's status in the module's policy
Status string `json:"status"`
@@ -698,6 +722,56 @@ type TestCounts struct {
Unit int64 `json:"unit,omitempty"`
}
// TestRun is one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u).
type TestRun struct {
// the id of the build seat's ask
Ask string `json:"ask,omitempty"`
// the share digest (design 47 G §724)
Digest string `json:"digest"`
Ended *time.Time `json:"ended,omitempty"`
// why the packages are not named: the run's output is gone
Gone string `json:"gone,omitempty"`
// the key, <candidate key>.<level>.<digest>, each part escaped as a key's name
Key string `json:"key"`
Level string `json:"level"`
// the node whose holder took the ask
Node string `json:"node,omitempty"`
Outcome string `json:"outcome,omitempty"`
// the run's output object, by its name in test-runs-objects
Output string `json:"output,omitempty"`
// the packages it ran, read from its output object
Packages []TestRunPackage `json:"packages,omitempty"`
// the steps reading the run
Readers []TestRunReader `json:"readers,omitempty"`
// the outcome's words
Said string `json:"said,omitempty"`
// the build seat asked
Seat string `json:"seat,omitempty"`
Started *time.Time `json:"started,omitempty"`
Tests *TestCounts `json:"tests,omitempty"`
}
// The values of TestRun's level.
const (
TestRunLevelLint = "lint"
TestRunLevelUnit = "unit"
TestRunLevelIntegration = "integration"
)
// TestRunPackage is one package a test run ran, with its digest (design 47 G §724).
type TestRunPackage struct {
Digest string `json:"digest"`
Package string `json:"package"`
}
// TestRunReader is a test step reading a test run (design 47 G §725).
type TestRunReader struct {
// its artifact, <module>:<artifact>
Artifact string `json:"artifact"`
// its pipeline's key
Pipeline string `json:"pipeline"`
}
// ToolchainUsed is the toolchain a build was made with: the language, the installed version and the build module's recipe.
type ToolchainUsed struct {
Language string `json:"language"`
@@ -708,6 +782,8 @@ type ToolchainUsed struct {
// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283).
type Wait struct {
Artifact string `json:"artifact,omitempty"`
// the step's own ask, where it has one (design 47 G §731)
Ask string `json:"ask,omitempty"`
// the watched branch, or the branch of a run on request
Branch string `json:"branch"`
// the commit the module's entry was last changed at
@@ -726,8 +802,10 @@ type Wait struct {
// the catalog graph revision of the branch the pipeline runs for
Revision int64 `json:"revision,omitempty"`
// the run on request's id, for a pipeline of a run; such a pipeline delivers nothing
Run string `json:"run,omitempty"`
Since time.Time `json:"since"`
Run string `json:"run,omitempty"`
// the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)
Shared []Shared `json:"shared,omitempty"`
Since time.Time `json:"since"`
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
Source int64 `json:"source,omitempty"`
// a pipeline step a wait inside a pipeline is in
@@ -977,6 +1055,44 @@ type AskAgainAnswer struct {
Result StepResult `json:"result"`
}
// TestRunsInput is what test-runs takes.
type TestRunsInput struct {
// the candidate's key
Candidate string `json:"candidate,omitempty"`
Digest string `json:"digest,omitempty"`
Level string `json:"level,omitempty"`
Module string `json:"module,omitempty"`
}
// The values of TestRunsInput's level.
const (
TestRunsInputLevelLint = "lint"
TestRunsInputLevelUnit = "unit"
TestRunsInputLevelIntegration = "integration"
)
// TestRunsAnswer is what test-runs answers.
type TestRunsAnswer struct {
Fill Fill `json:"fill"`
Runs []TestRun `json:"runs"`
}
// TestRunsDropInput is what test-runs-drop takes.
type TestRunsDropInput struct {
// the run's key in test-runs
Key string `json:"key"`
// why, written with the caller the runtime proves in the hand-act log
Why string `json:"why"`
}
// TestRunsDropAnswer is what test-runs-drop answers.
type TestRunsDropAnswer struct {
// the readers it ended
Ended []TestRunReader `json:"ended,omitempty"`
// the entry deleted
Run TestRun `json:"run"`
}
// PipelineStartedEvent is the body of the event pipeline-started.
type PipelineStartedEvent = PipelineStarted
@@ -37,6 +37,12 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.wait-held\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "the wait ended answered", subject: "mesh.seat.mesh-pipeline-runner.event.wait-ended.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"how\":\"answered\",\"module\":\"nats\",\"on\":\"person\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"since\":\"2026-11-10T10:10:00Z\",\"source\":3589,\"step\":\"approval\",\"v\":1,\"who\":\"the operator\"}", wire: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"how\":\"answered\",\"module\":\"nats\",\"on\":\"person\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"since\":\"2026-11-10T10:10:00Z\",\"source\":3589,\"step\":\"approval\",\"v\":1,\"who\":\"the operator\"}",
decode: func(b []byte) (any, error) { var v WaitEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.wait-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "C1/AC2 a test step started reading a test run, naming its run and ask", subject: "mesh.seat.mesh-pipeline-runner.event.step-started.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"at\":\"2026-11-17T10:00:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"outcome\":\"running\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\"},\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-17T10:00:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"outcome\":\"running\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\"},\"revision\":12,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v StepStartedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-started\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "C1/AC2 a test step ended with the run another step asked", subject: "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"asked-by\":{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"asked-by\":{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},\"revision\":12,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "C1 a composition step ended with mesh-wants' runs as they came", subject: "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"fingerprint\":\"fp-0123\",\"node\":\"laptop\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}],\"status\":\"required\",\"step\":\"composition\"},\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"fingerprint\":\"fp-0123\",\"node\":\"laptop\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}],\"status\":\"required\",\"step\":\"composition\"},\"revision\":12,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a pipeline kept", subject: "", given: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.11.nats\",\"fingerprint\":\"fp-0123\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.11.nats\",\"fingerprint\":\"fp-0123\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}",
decode: func(b []byte) (any, error) {
var v PipelinesByBranchRevisionModuleValue
@@ -67,6 +73,12 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v StandsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "kept answers by digest", subject: "", given: "{\"fill\":{\"bytes\":300,\"cap\":134217728,\"store\":\"kept\"},\"kept\":[{\"digest\":\"sha256:aa\",\"level\":\"unit\",\"package\":\"git.novox.be/novox/nats\",\"pipeline\":\"main.12.nats\"}]}", wire: "{\"fill\":{\"bytes\":300,\"cap\":134217728,\"store\":\"kept\"},\"kept\":[{\"digest\":\"sha256:aa\",\"level\":\"unit\",\"package\":\"git.novox.be/novox/nats\",\"pipeline\":\"main.12.nats\"}]}",
decode: func(b []byte) (any, error) { var v KeptAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1/AC3 stands answers a waiting test step with its ask and the run it waits on", subject: "", given: "{\"stands\":[{\"artifact\":\"nats:tools\",\"ask\":\"t1a2\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"on\":\"build node\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"since\":\"2026-11-17T10:00:00Z\",\"source\":3589,\"step\":\"unit\",\"v\":1}]}", wire: "{\"stands\":[{\"artifact\":\"nats:tools\",\"ask\":\"t1a2\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"on\":\"build node\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"since\":\"2026-11-17T10:00:00Z\",\"source\":3589,\"step\":\"unit\",\"v\":1}]}",
decode: func(b []byte) (any, error) { var v StandsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 test-runs answers each run with its packages, readers and the fill", subject: "", given: "{\"fill\":{\"bytes\":2048,\"cap\":16777216,\"store\":\"test-runs\"},\"runs\":[{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}},{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"gone\":\"the run's output is gone\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}]}", wire: "{\"fill\":{\"bytes\":2048,\"cap\":16777216,\"store\":\"test-runs\"},\"runs\":[{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}},{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"gone\":\"the run's output is gone\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}]}",
decode: func(b []byte) (any, error) { var v TestRunsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 test-runs-drop answers the run it deleted and the readers it ended", subject: "", given: "{\"ended\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"run\":{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}}", wire: "{\"ended\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"run\":{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}}",
decode: func(b []byte) (any, error) { var v TestRunsDropAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
}
for _, c := range cases {
v, err := c.decode([]byte(c.given))
@@ -15,12 +15,14 @@ import (
const Seat = "mesh-wants"
const (
VerbUpgrade = "upgrade"
VerbBus = "bus"
VerbRevision = "revision"
VerbEntry = "entry"
VerbDifference = "difference"
VerbInput = "input"
VerbUpgrade = "upgrade"
VerbBus = "bus"
VerbRevision = "revision"
VerbEntry = "entry"
VerbDifference = "difference"
VerbInput = "input"
VerbCompositions = "compositions"
VerbCompositionsDrop = "compositions-drop"
)
const (
@@ -44,7 +46,7 @@ const (
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
func Verbs() []string {
return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput}
return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput, VerbCompositions, VerbCompositionsDrop}
}
// Events is every event published on the seat's subject: a new copy each call.
@@ -125,7 +127,6 @@ const (
StateDifferences = "differences"
StateCompositions = "compositions"
StateAssignmentRevisions = "assignment-revisions"
StateInputs = "inputs"
)
// WantsByWantNodeModuleKey is the key of one value of the state wants under the form "want.<node>.<module>".
@@ -741,42 +742,6 @@ func DecodeAssignmentRevisionsByEntryDigestValue(raw []byte) (AssignmentRevision
return v, nil
}
// InputsByTimeDigestKey is the key of one value of the state inputs under the form "<time>.<digest>".
func InputsByTimeDigestKey(time, digest string) string { return time + "." + digest }
// ParseInputsByTimeDigestKey reads the parts of a key of the form "<time>.<digest>", in the order of the form; the last result is false for a key of another form.
func ParseInputsByTimeDigestKey(key string) (string, string, bool) {
rest := key
v0, after0, found0 := strings.Cut(rest, ".")
if !found0 || v0 == "" {
return "", "", false
}
rest = after0
if rest == "" {
return "", "", false
}
v2 := rest
rest = ""
if rest != "" {
return "", "", false
}
return v0, v2, true
}
// DecodeInputsByTimeDigestValue reads one value of the state inputs under the form "<time>.<digest>", refusing a field InputsByTimeDigestValue does not know and any text after the value.
func DecodeInputsByTimeDigestValue(raw []byte) (InputsByTimeDigestValue, error) {
var v InputsByTimeDigestValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
}
// Ask is an ask the assigner puts to a person: the router's ask whole, as the assigner composes it, and how many times it was written at its key.
type Ask struct {
// the seat operator-channel's ask, whole
@@ -867,6 +832,79 @@ type Change struct {
To string `json:"to,omitempty"`
}
// CompositionReader is a composition step reading a composition run: its step, as the runner asked it, with its ask's id (design 47 G §719, A-468).
type CompositionReader struct {
// the watched branch, or the branch of a run on request
Branch string `json:"branch"`
// for a rerun, the caller the runtime proved
By string `json:"by,omitempty"`
// the commit the step runs at
Commit string `json:"commit"`
// when the reader's step said its end; carried into a rerun's run, so a take-up tells a reader that passed from one that joined later (A-470m)
Ended *time.Time `json:"ended,omitempty"`
// the module's build-input fingerprint at the commit
Fingerprint string `json:"fingerprint"`
// the step's ask's id
ID string `json:"id"`
Module string `json:"module"`
// the runner's pipeline key, <branch>.<revision>.<module>, a branch escaped as a key's name is
Pipeline string `json:"pipeline"`
// the step is asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the catalog graph revision of the branch the pipeline runs for
Revision int64 `json:"revision,omitempty"`
// the run on request's id: its composition writes no want (ADR 0666 section 18)
Run string `json:"run,omitempty"`
// the source revision of the watched branch; absent for a run on request
Source int64 `json:"source,omitempty"`
Step string `json:"step"`
}
// The values of CompositionReader's step.
const (
CompositionReaderStepComposition = "composition"
)
// CompositionRunProgram is the composer program, chosen once when the run started.
type CompositionRunProgram struct {
Commit string `json:"commit,omitempty"`
Directory string `json:"directory,omitempty"`
// the node-engine fingerprint it was chosen from
Fingerprint string `json:"fingerprint,omitempty"`
Repository string `json:"repository,omitempty"`
}
// CompositionRun is one composition run of a candidate and a machine, a key of the assigner's state composition-runs (design 47 G §719, §728; A-469u).
type CompositionRun struct {
// the id of the build seat's run ask
Ask string `json:"ask,omitempty"`
// the candidate's key: <branch>.<revision>, or run-<id> for a run on request
Candidate string `json:"candidate"`
Ended *time.Time `json:"ended,omitempty"`
// the digest of the kept request
Input string `json:"input,omitempty"`
// the key, <candidate key>.<machine>, each part escaped as a key's name
Key string `json:"key"`
Machine string `json:"machine"`
Outcome string `json:"outcome,omitempty"`
// the composer program, chosen once when the run started
Program *CompositionRunProgram `json:"program,omitempty"`
// the composition steps reading the run
Readers []CompositionReader `json:"readers,omitempty"`
// the outcome's words
Said string `json:"said,omitempty"`
Started *time.Time `json:"started,omitempty"`
// that machine's composer verdict (design 47 G §721)
Verdict map[string]any `json:"verdict,omitempty"`
}
// The values of CompositionRun's outcome.
const (
CompositionRunOutcomePassed = "passed"
CompositionRunOutcomeFailed = "failed"
CompositionRunOutcomeCouldNotRun = "could not run"
)
// DifferenceRefusedItem is a type of the contract.
type DifferenceRefusedItem struct {
Module string `json:"module"`
@@ -944,6 +982,26 @@ type NodeEntryRef struct {
Node string `json:"node"`
}
// SharedAskedBy is the step that asked the run, for a test step reading another's run.
type SharedAskedBy struct {
// its artifact, <module>:<artifact>
Artifact string `json:"artifact"`
// its pipeline's key
Pipeline string `json:"pipeline"`
}
// Shared is a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself.
type Shared struct {
// the id of the run's ask of the build seat
Ask string `json:"ask"`
// the step that asked the run, for a test step reading another's run
AskedBy *SharedAskedBy `json:"asked-by,omitempty"`
// the machine composed, for a composition run
Machine string `json:"machine,omitempty"`
// the run's key: in the assigner's composition-runs, or the runner's test-runs
Run string `json:"run"`
}
// StepEnded is the composition step's end, said by the assigner on the node it runs on (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at.
type StepEnded struct {
// for a rerun, the caller the runtime proved
@@ -961,7 +1019,9 @@ type StepEnded struct {
Pipeline string `json:"pipeline"`
Rerun bool `json:"rerun,omitempty"`
// why it failed or could not run: the two modules owning one path, the part not heard
Said string `json:"said,omitempty"`
Said string `json:"said,omitempty"`
// per machine composed, the composition run the step read, its ask and so where its verdict was taken (design 47 G §723, §729)
Shared []Shared `json:"shared,omitempty"`
Started *time.Time `json:"started,omitempty"`
Step string `json:"step"`
// the layout version
@@ -1151,6 +1211,36 @@ type InputAnswer struct {
Digest string `json:"digest"`
}
// CompositionsInput is what compositions takes.
type CompositionsInput struct {
// the candidate's key
Candidate string `json:"candidate,omitempty"`
Machine string `json:"machine,omitempty"`
}
// CompositionsAnswer is what compositions answers.
type CompositionsAnswer struct {
Fill Fill `json:"fill"`
Runs []CompositionRun `json:"runs"`
}
// CompositionsDropInput is what compositions-drop takes.
type CompositionsDropInput struct {
// the candidate's key
Candidate string `json:"candidate"`
Machine string `json:"machine"`
// why, written with the caller the runtime proves in the hand-act log
Why string `json:"why"`
}
// CompositionsDropAnswer is what compositions-drop answers.
type CompositionsDropAnswer struct {
// the readers it ended
Ended []CompositionReader `json:"ended,omitempty"`
// the entry deleted
Run CompositionRun `json:"run"`
}
// StepAccept is the body of step.
type StepAccept = StepRun
@@ -1234,6 +1324,8 @@ type StepProgressEvent struct {
Asker string `json:"asker"`
At time.Time `json:"at"`
Phase string `json:"phase"`
// the composition runs the step waits on, by machine, each with its ask (design 47 G §723, §729)
Shared []Shared `json:"shared,omitempty"`
}
// The values of StepProgressEvent's phase.
@@ -1355,6 +1447,3 @@ type AssignmentRevisionsByCandidateGroupHeadsValue = AssignmentRevision
// AssignmentRevisionsByEntryDigestValue is one value of the state assignment-revisions under the key entry.<digest>.
type AssignmentRevisionsByEntryDigestValue = AssignmentEntry
// InputsByTimeDigestValue is one value of the state inputs under the key <time>.<digest>.
type InputsByTimeDigestValue = string
@@ -59,6 +59,10 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-held$")},
{name: "an outside wait ended by the operator's Go on", subject: "mesh.seat.mesh-wants.event.wait-ended", given: "{\"fingerprint\":\"src2:aaaa\",\"how\":\"go-on\",\"module\":\"nats\",\"node\":\"anchor\",\"on\":\"silent:home-server\",\"owner\":\"mesh-delivery-assigner\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"outside\"}", wire: "{\"fingerprint\":\"src2:aaaa\",\"how\":\"go-on\",\"module\":\"nats\",\"node\":\"anchor\",\"on\":\"silent:home-server\",\"owner\":\"mesh-delivery-assigner\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"outside\"}",
decode: func(b []byte) (any, error) { var v WaitEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-ended$")},
{name: "C1/AC1 a reading composition step ended, naming per machine the run it read and its ask", subject: "mesh.seat.mesh-wants.event.step-ended.laptop", given: "{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"node\":\"laptop\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"},{\"ask\":\"r4d5e6\",\"machine\":\"home-server\",\"run\":\"main.12.home-server\"}],\"step\":\"composition\",\"v\":1,\"warnings\":[\"anchor holds xclip: xclip and demo both own /etc/x\"]}", wire: "{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"node\":\"laptop\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"},{\"ask\":\"r4d5e6\",\"machine\":\"home-server\",\"run\":\"main.12.home-server\"}],\"step\":\"composition\",\"v\":1,\"warnings\":[\"anchor holds xclip: xclip and demo both own /etc/x\"]}",
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-ended\\.[^.*> ]+$")},
{name: "C1/AC1 a composition step's progress, naming the runs it waits on", subject: "mesh.seat.mesh-wants.event.step-progress.anchor", given: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-17T10:01:00Z\",\"phase\":\"run\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}]}", wire: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-17T10:01:00Z\",\"phase\":\"run\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}]}",
decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-progress\\.[^.*> ]+$")},
{name: "a want", subject: "", given: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}", wire: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}",
decode: func(b []byte) (any, error) {
var v WantsByWantNodeModuleValue
@@ -141,6 +145,12 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v InputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "input answers how many chunks carry the exact bytes", subject: "", given: "{\"chunks\":3,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", wire: "{\"chunks\":3,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}",
decode: func(b []byte) (any, error) { var v InputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 compositions answers each run with its readers, verdict and the fill", subject: "", given: "{\"fill\":{\"bytes\":4096,\"cap\":16777216,\"store\":\"composition-runs\"},\"runs\":[{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}]}", wire: "{\"fill\":{\"bytes\":4096,\"cap\":16777216,\"store\":\"composition-runs\"},\"runs\":[{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}]}",
decode: func(b []byte) (any, error) { var v CompositionsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 compositions-drop takes a candidate, a machine and why", subject: "", given: "{\"candidate\":\"main.12\",\"machine\":\"anchor\",\"why\":\"its ask never came back\"}", wire: "{\"candidate\":\"main.12\",\"machine\":\"anchor\",\"why\":\"its ask never came back\"}",
decode: func(b []byte) (any, error) { var v CompositionsDropInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 compositions-drop answers the run it deleted and the readers it ended", subject: "", given: "{\"ended\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"run\":{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}}", wire: "{\"ended\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"run\":{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}}",
decode: func(b []byte) (any, error) { var v CompositionsDropAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
}
for _, c := range cases {
v, err := c.decode([]byte(c.given))
@@ -313,5 +313,104 @@
"how": "answered",
"fingerprint": "fp-0123"
}
},
{
"name": "C1/AC2 a test step started reading a test run, naming its run and ask",
"of": "event:step-started",
"subject": "mesh.seat.mesh-pipeline-runner.event.step-started.nats.main.0123456789abcdef0123456789abcdef01234567",
"given": {
"pipeline": "main.12.nats",
"module": "nats",
"branch": "main",
"revision": 12,
"source": 3589,
"commit": "0123456789abcdef0123456789abcdef01234567",
"v": 1,
"fingerprint": "fp-0123",
"result": {
"step": "unit",
"artifact": "nats:tools",
"status": "required",
"outcome": "running",
"fingerprint": "fp-0123",
"seat": "node-build-go",
"shared": [
{
"run": "main.12.unit.d1",
"ask": "t1a2"
}
]
},
"at": "2026-11-17T10:00:00Z"
}
},
{
"name": "C1/AC2 a test step ended with the run another step asked",
"of": "event:step-ended",
"subject": "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567",
"given": {
"pipeline": "main.12.nats",
"module": "nats",
"branch": "main",
"revision": 12,
"source": 3589,
"commit": "0123456789abcdef0123456789abcdef01234567",
"v": 1,
"fingerprint": "fp-0123",
"result": {
"step": "unit",
"artifact": "nats:tools",
"status": "required",
"outcome": "failed",
"fingerprint": "fp-0123",
"seat": "node-build-go",
"node": "anchor",
"tests": {
"unit": 42
},
"shared": [
{
"run": "main.12.unit.d1",
"ask": "t1a2",
"asked-by": {
"pipeline": "main.12.nats",
"artifact": "nats:program"
}
}
]
},
"at": "2026-11-17T10:05:00Z"
}
},
{
"name": "C1 a composition step ended with mesh-wants' runs as they came",
"of": "event:step-ended",
"subject": "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567",
"given": {
"pipeline": "main.12.nats",
"module": "nats",
"branch": "main",
"revision": 12,
"source": 3589,
"commit": "0123456789abcdef0123456789abcdef01234567",
"v": 1,
"fingerprint": "fp-0123",
"result": {
"step": "composition",
"status": "required",
"outcome": "passed",
"fingerprint": "fp-0123",
"seat": "mesh-wants",
"node": "laptop",
"shared": [
{
"machine": "anchor",
"run": "main.12.anchor",
"ask": "r1a2b3"
}
]
},
"at": "2026-11-17T10:05:00Z"
}
}
]
@@ -143,5 +143,154 @@
"cap": 134217728
}
}
},
{
"name": "C1/AC3 stands answers a waiting test step with its ask and the run it waits on",
"of": "verb:stands.answer",
"given": {
"stands": [
{
"pipeline": "main.12.nats",
"module": "nats",
"branch": "main",
"revision": 12,
"source": 3589,
"commit": "0123456789abcdef0123456789abcdef01234567",
"v": 1,
"fingerprint": "fp-0123",
"step": "unit",
"artifact": "nats:tools",
"on": "build node",
"since": "2026-11-17T10:00:00Z",
"ask": "t1a2",
"shared": [
{
"run": "main.12.unit.d1",
"ask": "t1a2"
}
]
}
]
}
},
{
"name": "C1 test-runs answers each run with its packages, readers and the fill",
"of": "verb:test-runs.answer",
"given": {
"runs": [
{
"key": "main.12.unit.d1",
"level": "unit",
"digest": "d1",
"seat": "node-build-go",
"ask": "t1a2",
"node": "anchor",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"pipeline": "main.12.nats",
"artifact": "nats:program"
},
{
"pipeline": "main.12.nats",
"artifact": "nats:tools"
}
],
"ended": "2026-11-17T10:05:00Z",
"outcome": "failed",
"said": "1 test failed",
"tests": {
"unit": 42
},
"output": "t1a2",
"packages": [
{
"package": "git.novox.be/novox/nats/internal/x",
"digest": "sha256:aa"
}
]
},
{
"key": "main.12.unit.d1",
"level": "unit",
"digest": "d1",
"seat": "node-build-go",
"ask": "t1a2",
"node": "anchor",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"pipeline": "main.12.nats",
"artifact": "nats:program"
},
{
"pipeline": "main.12.nats",
"artifact": "nats:tools"
}
],
"ended": "2026-11-17T10:05:00Z",
"outcome": "failed",
"said": "1 test failed",
"tests": {
"unit": 42
},
"output": "t1a2",
"gone": "the run's output is gone"
}
],
"fill": {
"store": "test-runs",
"bytes": 2048,
"cap": 16777216
}
}
},
{
"name": "C1 test-runs-drop answers the run it deleted and the readers it ended",
"of": "verb:test-runs-drop.answer",
"given": {
"run": {
"key": "main.12.unit.d1",
"level": "unit",
"digest": "d1",
"seat": "node-build-go",
"ask": "t1a2",
"node": "anchor",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"pipeline": "main.12.nats",
"artifact": "nats:program"
},
{
"pipeline": "main.12.nats",
"artifact": "nats:tools"
}
],
"ended": "2026-11-17T10:05:00Z",
"outcome": "failed",
"said": "1 test failed",
"tests": {
"unit": 42
},
"output": "t1a2",
"packages": [
{
"package": "git.novox.be/novox/nats/internal/x",
"digest": "sha256:aa"
}
]
},
"ended": [
{
"pipeline": "main.12.nats",
"artifact": "nats:program"
},
{
"pipeline": "main.12.nats",
"artifact": "nats:tools"
}
]
}
}
]
@@ -495,6 +495,93 @@
]
},
"class": "act"
},
{
"name": "test-runs",
"class": "read",
"description": "Each test run the runner keeps (design 47 G §728): its packages read from its output object, ask, node, readers, outcome and counts, and the state's fill against its cap; with candidate, level, digest or module, only those.",
"input": {
"type": "object",
"properties": {
"candidate": {
"type": "string",
"description": "the candidate's key"
},
"level": {
"type": "string",
"enum": [
"lint",
"unit",
"integration"
]
},
"digest": {
"type": "string"
},
"module": {
"type": "string"
}
}
},
"answer": {
"type": "object",
"properties": {
"runs": {
"type": "array",
"items": {
"$ref": "#/defs/TestRun"
}
},
"fill": {
"$ref": "#/defs/Fill"
}
},
"required": [
"runs",
"fill"
]
}
},
{
"name": "test-runs-drop",
"class": "act",
"description": "Drop one test run by hand (design 47 G §728): the act and why are written in the hand-act log first, then the run's ask is dropped, each reader ended could not run naming the drop, and the key deleted with the run's output object.",
"input": {
"type": "object",
"properties": {
"key": {
"type": "string",
"description": "the run's key in test-runs"
},
"why": {
"type": "string",
"description": "why, written with the caller the runtime proves in the hand-act log"
}
},
"required": [
"key",
"why"
]
},
"answer": {
"type": "object",
"properties": {
"run": {
"$ref": "#/defs/TestRun",
"description": "the entry deleted"
},
"ended": {
"type": "array",
"items": {
"$ref": "#/defs/TestRunReader"
},
"description": "the readers it ended"
}
},
"required": [
"run"
]
}
}
],
"events": [
@@ -887,6 +974,13 @@
"ended": {
"type": "string",
"format": "date-time"
},
"shared": {
"type": "array",
"items": {
"$ref": "#/defs/Shared"
},
"description": "the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t)"
}
},
"description": "one step of a pipeline: its status in the policy and its outcome (design 47 *Pipeline steps and the policy*)",
@@ -1494,6 +1588,17 @@
"how": {
"type": "string",
"description": "for its end, how it ended: answered, ran, superseded"
},
"ask": {
"type": "string",
"description": "the step's own ask, where it has one (design 47 G §731)"
},
"shared": {
"type": "array",
"items": {
"$ref": "#/defs/Shared"
},
"description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)"
}
},
"description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283)",
@@ -1577,6 +1682,159 @@
"what",
"at"
]
},
"Shared": {
"type": "object",
"description": "a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself",
"properties": {
"machine": {
"type": "string",
"description": "the machine composed, for a composition run"
},
"run": {
"type": "string",
"description": "the run's key: in the assigner's composition-runs, or the runner's test-runs"
},
"ask": {
"type": "string",
"description": "the id of the run's ask of the build seat"
},
"asked-by": {
"type": "object",
"description": "the step that asked the run, for a test step reading another's run",
"properties": {
"pipeline": {
"type": "string",
"description": "its pipeline's key"
},
"artifact": {
"type": "string",
"description": "its artifact, <module>:<artifact>"
}
},
"required": [
"pipeline",
"artifact"
]
}
},
"required": [
"run",
"ask"
]
},
"TestRunReader": {
"type": "object",
"description": "a test step reading a test run (design 47 G §725)",
"properties": {
"pipeline": {
"type": "string",
"description": "its pipeline's key"
},
"artifact": {
"type": "string",
"description": "its artifact, <module>:<artifact>"
}
},
"required": [
"pipeline",
"artifact"
]
},
"TestRunPackage": {
"type": "object",
"description": "one package a test run ran, with its digest (design 47 G §724)",
"properties": {
"package": {
"type": "string"
},
"digest": {
"type": "string"
}
},
"required": [
"package",
"digest"
]
},
"TestRun": {
"type": "object",
"description": "one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u)",
"properties": {
"key": {
"type": "string",
"description": "the key, <candidate key>.<level>.<digest>, each part escaped as a key's name"
},
"level": {
"type": "string",
"enum": [
"lint",
"unit",
"integration"
]
},
"digest": {
"type": "string",
"description": "the share digest (design 47 G §724)"
},
"seat": {
"type": "string",
"description": "the build seat asked"
},
"ask": {
"type": "string",
"description": "the id of the build seat's ask"
},
"node": {
"type": "string",
"description": "the node whose holder took the ask"
},
"started": {
"type": "string",
"format": "date-time"
},
"readers": {
"type": "array",
"items": {
"$ref": "#/defs/TestRunReader"
},
"description": "the steps reading the run"
},
"ended": {
"type": "string",
"format": "date-time"
},
"outcome": {
"type": "string"
},
"said": {
"type": "string",
"description": "the outcome's words"
},
"tests": {
"$ref": "#/defs/TestCounts"
},
"output": {
"type": "string",
"description": "the run's output object, by its name in test-runs-objects"
},
"packages": {
"type": "array",
"items": {
"$ref": "#/defs/TestRunPackage"
},
"description": "the packages it ran, read from its output object"
},
"gone": {
"type": "string",
"description": "why the packages are not named: the run's output is gone"
}
},
"required": [
"key",
"level",
"digest"
]
}
}
}
@@ -284,5 +284,54 @@
"since": "2026-11-11T10:00:00Z",
"how": "go-on"
}
},
{
"name": "C1/AC1 a reading composition step ended, naming per machine the run it read and its ask",
"of": "event:step-ended",
"subject": "mesh.seat.mesh-wants.event.step-ended.laptop",
"given": {
"v": 1,
"id": "m2kx9a1b3c4d6c",
"pipeline": "main.12.nats",
"step": "composition",
"module": "nats",
"commit": "0123456789abcdef0123456789abcdef01234567",
"fingerprint": "fp-0123",
"node": "laptop",
"outcome": "passed",
"warnings": [
"anchor holds xclip: xclip and demo both own /etc/x"
],
"shared": [
{
"machine": "anchor",
"run": "main.12.anchor",
"ask": "r1a2b3"
},
{
"machine": "home-server",
"run": "main.12.home-server",
"ask": "r4d5e6"
}
]
}
},
{
"name": "C1/AC1 a composition step's progress, naming the runs it waits on",
"of": "event:step-progress",
"subject": "mesh.seat.mesh-wants.event.step-progress.anchor",
"given": {
"asker": "assigner",
"ask": "a1b2",
"phase": "run",
"at": "2026-11-17T10:01:00Z",
"shared": [
{
"machine": "anchor",
"run": "main.12.anchor",
"ask": "r1a2b3"
}
]
}
}
]
@@ -175,5 +175,109 @@
"chunks": 3,
"digest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
}
},
{
"name": "C1 compositions answers each run with its readers, verdict and the fill",
"of": "verb:compositions.answer",
"given": {
"runs": [
{
"key": "main.12.anchor",
"candidate": "main.12",
"machine": "anchor",
"program": {
"repository": "novox/mesh-host",
"commit": "0123456789abcdef0123456789abcdef01234567",
"directory": "cmd/mesh-compose",
"fingerprint": "src2:aaaa"
},
"input": "sha256:0123",
"ask": "r1a2b3",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"id": "m2kx9a1b3c4d6c",
"pipeline": "main.12.nats",
"step": "composition",
"module": "nats",
"branch": "main",
"commit": "0123456789abcdef0123456789abcdef01234567",
"revision": 12,
"fingerprint": "fp-0123"
}
],
"ended": "2026-11-17T10:03:00Z",
"outcome": "passed",
"verdict": {
"composes": true,
"blamed": []
}
}
],
"fill": {
"store": "composition-runs",
"bytes": 4096,
"cap": 16777216
}
}
},
{
"name": "C1 compositions-drop takes a candidate, a machine and why",
"of": "verb:compositions-drop.input",
"given": {
"candidate": "main.12",
"machine": "anchor",
"why": "its ask never came back"
}
},
{
"name": "C1 compositions-drop answers the run it deleted and the readers it ended",
"of": "verb:compositions-drop.answer",
"given": {
"run": {
"key": "main.12.anchor",
"candidate": "main.12",
"machine": "anchor",
"program": {
"repository": "novox/mesh-host",
"commit": "0123456789abcdef0123456789abcdef01234567",
"directory": "cmd/mesh-compose",
"fingerprint": "src2:aaaa"
},
"input": "sha256:0123",
"ask": "r1a2b3",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"id": "m2kx9a1b3c4d6c",
"pipeline": "main.12.nats",
"step": "composition",
"module": "nats",
"branch": "main",
"commit": "0123456789abcdef0123456789abcdef01234567",
"revision": 12,
"fingerprint": "fp-0123"
}
],
"ended": "2026-11-17T10:03:00Z",
"outcome": "passed",
"verdict": {
"composes": true,
"blamed": []
}
},
"ended": [
{
"id": "m2kx9a1b3c4d6c",
"pipeline": "main.12.nats",
"step": "composition",
"module": "nats",
"branch": "main",
"commit": "0123456789abcdef0123456789abcdef01234567",
"revision": 12,
"fingerprint": "fp-0123"
}
]
}
}
]
@@ -202,6 +202,86 @@
]
},
"chunks": true
},
{
"name": "compositions",
"class": "read",
"description": "Each composition run the assigner keeps (design 47 G \u00a7728): its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those.",
"input": {
"type": "object",
"properties": {
"candidate": {
"type": "string",
"description": "the candidate's key"
},
"machine": {
"type": "string"
}
}
},
"answer": {
"type": "object",
"properties": {
"runs": {
"type": "array",
"items": {
"$ref": "#/defs/CompositionRun"
}
},
"fill": {
"$ref": "#/defs/Fill"
}
},
"required": [
"runs",
"fill"
]
}
},
{
"name": "compositions-drop",
"class": "act",
"description": "Drop one composition run by hand (design 47 G \u00a7728): the act and why are written in the hand-act log first, then the run's ask is dropped, each reader ended could not run naming the drop, and the key deleted.",
"input": {
"type": "object",
"properties": {
"candidate": {
"type": "string",
"description": "the candidate's key"
},
"machine": {
"type": "string"
},
"why": {
"type": "string",
"description": "why, written with the caller the runtime proves in the hand-act log"
}
},
"required": [
"candidate",
"machine",
"why"
]
},
"answer": {
"type": "object",
"properties": {
"run": {
"$ref": "#/defs/CompositionRun",
"description": "the entry deleted"
},
"ended": {
"type": "array",
"items": {
"$ref": "#/defs/CompositionReader"
},
"description": "the readers it ended"
}
},
"required": [
"run"
]
}
}
],
"accepts": [
@@ -470,6 +550,13 @@
"at": {
"type": "string",
"format": "date-time"
},
"shared": {
"type": "array",
"items": {
"$ref": "#/defs/Shared"
},
"description": "the composition runs the step waits on, by machine, each with its ask (design 47 G \u00a7723, \u00a7729)"
}
},
"required": [
@@ -808,22 +895,6 @@
"description": "an entry, written once, removed when no kept revision lists it"
}
]
},
{
"name": "inputs",
"template": "state",
"history": 1,
"description": "a step's input, named by its write time and its sha256; a value over one chunk is held in the state's objects store (design 47 G sections 354, 712)",
"key-forms": [
{
"key": "<time>.<digest>",
"value": {
"type": "string",
"description": "the input's bytes, base64"
},
"description": "one input"
}
]
}
],
"defs": {
@@ -1434,6 +1505,13 @@
"ended": {
"type": "string",
"format": "date-time"
},
"shared": {
"type": "array",
"items": {
"$ref": "#/defs/Shared"
},
"description": "per machine composed, the composition run the step read, its ask and so where its verdict was taken (design 47 G \u00a7723, \u00a7729)"
}
},
"required": [
@@ -1488,6 +1566,195 @@
"required": [
"ask"
]
},
"Shared": {
"type": "object",
"description": "a run whose result a step took or waits on, asked once for every step that reads it (design 47 G \u00a7729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself",
"properties": {
"machine": {
"type": "string",
"description": "the machine composed, for a composition run"
},
"run": {
"type": "string",
"description": "the run's key: in the assigner's composition-runs, or the runner's test-runs"
},
"ask": {
"type": "string",
"description": "the id of the run's ask of the build seat"
},
"asked-by": {
"type": "object",
"description": "the step that asked the run, for a test step reading another's run",
"properties": {
"pipeline": {
"type": "string",
"description": "its pipeline's key"
},
"artifact": {
"type": "string",
"description": "its artifact, <module>:<artifact>"
}
},
"required": [
"pipeline",
"artifact"
]
}
},
"required": [
"run",
"ask"
]
},
"CompositionReader": {
"type": "object",
"description": "a composition step reading a composition run: its step, as the runner asked it, with its ask's id (design 47 G \u00a7719, A-468)",
"properties": {
"id": {
"type": "string",
"description": "the step's ask's id"
},
"pipeline": {
"type": "string",
"description": "the runner's pipeline key, <branch>.<revision>.<module>, a branch escaped as a key's name is"
},
"step": {
"type": "string",
"enum": [
"composition"
]
},
"module": {
"type": "string"
},
"branch": {
"type": "string",
"description": "the watched branch, or the branch of a run on request"
},
"commit": {
"type": "string",
"description": "the commit the step runs at"
},
"revision": {
"type": "integer",
"description": "the catalog graph revision of the branch the pipeline runs for"
},
"source": {
"type": "integer",
"description": "the source revision of the watched branch; absent for a run on request"
},
"fingerprint": {
"type": "string",
"description": "the module's build-input fingerprint at the commit"
},
"run": {
"type": "string",
"description": "the run on request's id: its composition writes no want (ADR 0666 section 18)"
},
"rerun": {
"type": "boolean",
"description": "the step is asked again by the runner's rerun"
},
"by": {
"type": "string",
"description": "for a rerun, the caller the runtime proved"
},
"ended": {
"type": "string",
"format": "date-time",
"description": "when the reader's step said its end; carried into a rerun's run, so a take-up tells a reader that passed from one that joined later (A-470m)"
}
},
"required": [
"id",
"pipeline",
"step",
"module",
"branch",
"commit",
"fingerprint"
]
},
"CompositionRun": {
"type": "object",
"description": "one composition run of a candidate and a machine, a key of the assigner's state composition-runs (design 47 G \u00a7719, \u00a7728; A-469u)",
"properties": {
"key": {
"type": "string",
"description": "the key, <candidate key>.<machine>, each part escaped as a key's name"
},
"candidate": {
"type": "string",
"description": "the candidate's key: <branch>.<revision>, or run-<id> for a run on request"
},
"machine": {
"type": "string"
},
"program": {
"type": "object",
"description": "the composer program, chosen once when the run started",
"properties": {
"repository": {
"type": "string"
},
"commit": {
"type": "string"
},
"directory": {
"type": "string"
},
"fingerprint": {
"type": "string",
"description": "the node-engine fingerprint it was chosen from"
}
}
},
"input": {
"type": "string",
"description": "the digest of the kept request"
},
"ask": {
"type": "string",
"description": "the id of the build seat's run ask"
},
"started": {
"type": "string",
"format": "date-time"
},
"readers": {
"type": "array",
"items": {
"$ref": "#/defs/CompositionReader"
},
"description": "the composition steps reading the run"
},
"ended": {
"type": "string",
"format": "date-time"
},
"outcome": {
"type": "string",
"enum": [
"passed",
"failed",
"could not run"
]
},
"said": {
"type": "string",
"description": "the outcome's words"
},
"verdict": {
"type": "object",
"description": "that machine's composer verdict (design 47 G \u00a7721)"
}
},
"required": [
"key",
"candidate",
"machine"
]
}
},
"acts": [
@@ -17,6 +17,8 @@ export declare const verbs: {
readonly accept: "accept";
readonly cancel: "cancel";
readonly askAgain: "ask-again";
readonly testRuns: "test-runs";
readonly testRunsDrop: "test-runs-drop";
};
export declare const events: {
@@ -316,6 +318,26 @@ export interface RunStarted {
"v": number;
}
/** the step that asked the run, for a test step reading another's run */
export interface SharedAskedBy {
/** its artifact, <module>:<artifact> */
"artifact": string;
/** its pipeline's key */
"pipeline": string;
}
/** a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself */
export interface Shared {
/** the id of the run's ask of the build seat */
"ask": string;
/** the step that asked the run, for a test step reading another's run */
"asked-by"?: SharedAskedBy;
/** the machine composed, for a composition run */
"machine"?: string;
/** the run's key: in the assigner's composition-runs, or the runner's test-runs */
"run": string;
}
/** a step's start or end: the step, its status, its outcome, its fingerprint, its test counts, what a carried result was carried from, and for a rerun that it is one and its proved caller; a bypassed step too */
export interface StepEvent {
"at"?: string;
@@ -366,6 +388,8 @@ export interface StepResult {
"rerun"?: boolean;
/** the seat that serves the step, whose holder alone may end it (ADR 0666 section 21) */
"seat"?: string;
/** the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t) */
"shared"?: Shared[];
"started"?: string;
/** the step's status in the module's policy */
"status": "required" | "optional" | "bypass";
@@ -381,6 +405,49 @@ export interface TestCounts {
"unit"?: number;
}
/** one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u) */
export interface TestRun {
/** the id of the build seat's ask */
"ask"?: string;
/** the share digest (design 47 G §724) */
"digest": string;
"ended"?: string;
/** why the packages are not named: the run's output is gone */
"gone"?: string;
/** the key, <candidate key>.<level>.<digest>, each part escaped as a key's name */
"key": string;
"level": "lint" | "unit" | "integration";
/** the node whose holder took the ask */
"node"?: string;
"outcome"?: string;
/** the run's output object, by its name in test-runs-objects */
"output"?: string;
/** the packages it ran, read from its output object */
"packages"?: TestRunPackage[];
/** the steps reading the run */
"readers"?: TestRunReader[];
/** the outcome's words */
"said"?: string;
/** the build seat asked */
"seat"?: string;
"started"?: string;
"tests"?: TestCounts;
}
/** one package a test run ran, with its digest (design 47 G §724) */
export interface TestRunPackage {
"digest": string;
"package": string;
}
/** a test step reading a test run (design 47 G §725) */
export interface TestRunReader {
/** its artifact, <module>:<artifact> */
"artifact": string;
/** its pipeline's key */
"pipeline": string;
}
/** the toolchain a build was made with: the language, the installed version and the build module's recipe */
export interface ToolchainUsed {
"language": string;
@@ -391,6 +458,8 @@ export interface ToolchainUsed {
/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283) */
export interface Wait {
"artifact"?: string;
/** the step's own ask, where it has one (design 47 G §731) */
"ask"?: string;
/** the watched branch, or the branch of a run on request */
"branch": string;
/** the commit the module's entry was last changed at */
@@ -410,6 +479,8 @@ export interface Wait {
"revision"?: number;
/** the run on request's id, for a pipeline of a run; such a pipeline delivers nothing */
"run"?: string;
/** the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v) */
"shared"?: Shared[];
"since": string;
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
"source"?: number;
@@ -594,6 +665,37 @@ export interface AskAgainAnswer {
"result": StepResult;
}
/** is what test-runs takes. */
export interface TestRunsInput {
/** the candidate's key */
"candidate"?: string;
"digest"?: string;
"level"?: "lint" | "unit" | "integration";
"module"?: string;
}
/** is what test-runs answers. */
export interface TestRunsAnswer {
"fill": Fill;
"runs": TestRun[];
}
/** is what test-runs-drop takes. */
export interface TestRunsDropInput {
/** the run's key in test-runs */
"key": string;
/** why, written with the caller the runtime proves in the hand-act log */
"why": string;
}
/** is what test-runs-drop answers. */
export interface TestRunsDropAnswer {
/** the readers it ended */
"ended"?: TestRunReader[];
/** the entry deleted */
"run": TestRun;
}
/** is the body of the event pipeline-started. */
export type PipelineStartedEvent = PipelineStarted;
@@ -17,6 +17,8 @@ export const verbs = {
accept: "accept",
cancel: "cancel",
askAgain: "ask-again",
testRuns: "test-runs",
testRunsDrop: "test-runs-drop",
};
export const events = {
+117 -10
View File
@@ -11,6 +11,8 @@ export declare const verbs: {
readonly entry: "entry";
readonly difference: "difference";
readonly input: "input";
readonly compositions: "compositions";
readonly compositionsDrop: "compositions-drop";
};
export declare const accepts: {
@@ -78,7 +80,6 @@ export declare const states: {
readonly differences: "differences";
readonly compositions: "compositions";
readonly assignmentRevisions: "assignment-revisions";
readonly inputs: "inputs";
};
/** The key of one value of the state wants under the form "want.<node>.<module>". */
@@ -171,12 +172,6 @@ export declare function assignmentRevisionsByEntryDigestKey(digest: string): str
/** The parts of a key of the form "entry.<digest>", or undefined for a key of another form. */
export declare function parseAssignmentRevisionsByEntryDigestKey(_key: string): { digest: string } | undefined;
/** The key of one value of the state inputs under the form "<time>.<digest>". */
export declare function inputsByTimeDigestKey(time: string, digest: string): string;
/** The parts of a key of the form "<time>.<digest>", or undefined for a key of another form. */
export declare function parseInputsByTimeDigestKey(_key: string): { time: string; digest: string } | undefined;
/** an ask the assigner puts to a person: the router's ask whole, as the assigner composes it, and how many times it was written at its key */
export interface Ask {
/** the seat operator-channel's ask, whole */
@@ -265,6 +260,67 @@ export interface Change {
"to"?: string;
}
/** a composition step reading a composition run: its step, as the runner asked it, with its ask's id (design 47 G §719, A-468) */
export interface CompositionReader {
/** the watched branch, or the branch of a run on request */
"branch": string;
/** for a rerun, the caller the runtime proved */
"by"?: string;
/** the commit the step runs at */
"commit": string;
/** when the reader's step said its end; carried into a rerun's run, so a take-up tells a reader that passed from one that joined later (A-470m) */
"ended"?: string;
/** the module's build-input fingerprint at the commit */
"fingerprint": string;
/** the step's ask's id */
"id": string;
"module": string;
/** the runner's pipeline key, <branch>.<revision>.<module>, a branch escaped as a key's name is */
"pipeline": string;
/** the step is asked again by the runner's rerun */
"rerun"?: boolean;
/** the catalog graph revision of the branch the pipeline runs for */
"revision"?: number;
/** the run on request's id: its composition writes no want (ADR 0666 section 18) */
"run"?: string;
/** the source revision of the watched branch; absent for a run on request */
"source"?: number;
"step": "composition";
}
/** the composer program, chosen once when the run started */
export interface CompositionRunProgram {
"commit"?: string;
"directory"?: string;
/** the node-engine fingerprint it was chosen from */
"fingerprint"?: string;
"repository"?: string;
}
/** one composition run of a candidate and a machine, a key of the assigner's state composition-runs (design 47 G §719, §728; A-469u) */
export interface CompositionRun {
/** the id of the build seat's run ask */
"ask"?: string;
/** the candidate's key: <branch>.<revision>, or run-<id> for a run on request */
"candidate": string;
"ended"?: string;
/** the digest of the kept request */
"input"?: string;
/** the key, <candidate key>.<machine>, each part escaped as a key's name */
"key": string;
"machine": string;
"outcome"?: "passed" | "failed" | "could not run";
/** the composer program, chosen once when the run started */
"program"?: CompositionRunProgram;
/** the composition steps reading the run */
"readers"?: CompositionReader[];
/** the outcome's words */
"said"?: string;
"started"?: string;
/** that machine's composer verdict (design 47 G §721) */
"verdict"?: Record<string, unknown>;
}
export interface DifferenceRefusedItem {
"module": string;
"why": string;
@@ -340,6 +396,26 @@ export interface NodeEntryRef {
"node": string;
}
/** the step that asked the run, for a test step reading another's run */
export interface SharedAskedBy {
/** its artifact, <module>:<artifact> */
"artifact": string;
/** its pipeline's key */
"pipeline": string;
}
/** a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself */
export interface Shared {
/** the id of the run's ask of the build seat */
"ask": string;
/** the step that asked the run, for a test step reading another's run */
"asked-by"?: SharedAskedBy;
/** the machine composed, for a composition run */
"machine"?: string;
/** the run's key: in the assigner's composition-runs, or the runner's test-runs */
"run": string;
}
/** the composition step's end, said by the assigner on the node it runs on (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at */
export interface StepEnded {
/** for a rerun, the caller the runtime proved */
@@ -358,6 +434,8 @@ export interface StepEnded {
"rerun"?: boolean;
/** why it failed or could not run: the two modules owning one path, the part not heard */
"said"?: string;
/** per machine composed, the composition run the step read, its ask and so where its verdict was taken (design 47 G §723, §729) */
"shared"?: Shared[];
"started"?: string;
"step": "composition";
/** the layout version */
@@ -506,6 +584,36 @@ export interface InputAnswer {
"digest": string;
}
/** is what compositions takes. */
export interface CompositionsInput {
/** the candidate's key */
"candidate"?: string;
"machine"?: string;
}
/** is what compositions answers. */
export interface CompositionsAnswer {
"fill": Fill;
"runs": CompositionRun[];
}
/** is what compositions-drop takes. */
export interface CompositionsDropInput {
/** the candidate's key */
"candidate": string;
"machine": string;
/** why, written with the caller the runtime proves in the hand-act log */
"why": string;
}
/** is what compositions-drop answers. */
export interface CompositionsDropAnswer {
/** the readers it ended */
"ended"?: CompositionReader[];
/** the entry deleted */
"run": CompositionRun;
}
/** is the body of step. */
export type StepAccept = StepRun;
@@ -589,6 +697,8 @@ export interface StepProgressEvent {
"asker": string;
"at": string;
"phase": "fetch" | "build" | "run" | "report";
/** the composition runs the step waits on, by machine, each with its ask (design 47 G §723, §729) */
"shared"?: Shared[];
}
/** is one value of the state wants under the key want.<node>.<module>. */
@@ -692,6 +802,3 @@ export type AssignmentRevisionsByCandidateGroupHeadsValue = AssignmentRevision;
/** is one value of the state assignment-revisions under the key entry.<digest>. */
export type AssignmentRevisionsByEntryDigestValue = AssignmentEntry;
/** is one value of the state inputs under the key <time>.<digest>. */
export type InputsByTimeDigestValue = string;
@@ -11,6 +11,8 @@ export const verbs = {
entry: "entry",
difference: "difference",
input: "input",
compositions: "compositions",
compositionsDrop: "compositions-drop",
};
export const accepts = {
@@ -84,7 +86,6 @@ export const states = {
differences: "differences",
compositions: "compositions",
assignmentRevisions: "assignment-revisions",
inputs: "inputs",
};
/** The key of one value of the state wants under the form "want.<node>.<module>". */
@@ -403,22 +404,3 @@ export function parseAssignmentRevisionsByEntryDigestKey(_key) {
return { digest };
}
/** The key of one value of the state inputs under the form "<time>.<digest>". */
export function inputsByTimeDigestKey(time, digest) {
return time + "." + digest;
}
/** The parts of a key of the form "<time>.<digest>", or undefined for a key of another form. */
export function parseInputsByTimeDigestKey(_key) {
let _rest = _key;
const _at0 = _rest.indexOf(".");
if (_at0 <= 0) return undefined;
const time = _rest.slice(0, _at0);
_rest = _rest.slice(_at0 + 1);
if (_rest === "") return undefined;
const digest = _rest;
_rest = "";
if (_rest !== "") return undefined;
return { time, digest };
}
@@ -46,7 +46,7 @@ func as[T any](args map[string]any) (T, error) {
return in, nil
}
// Tools are the runner's read verbs over its states pipelines and kept.
// Tools are the runner's read verbs over its states pipelines and kept; test-runs is the process's (A-470i).
func Tools(r pipeline.Reads) []stdio.Tool {
runs := map[string]func(map[string]any) (any, error){
seat.VerbShow: func(a map[string]any) (any, error) {
@@ -152,6 +152,7 @@ func main() {
tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask},
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...)
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
tools = append(tools, TestRunsTools(stdio.Ask)...)
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
@@ -0,0 +1,43 @@
package main
import (
"encoding/json"
"errors"
"strings"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// TestRunsTools are runner_test_runs and runner_test_runs_drop (design 47 G §728; A-469f, A-469u): each calls its seat verb
// over the bus, both answered by the runner's process, which holds test-runs-cap-mib (A-469g, A-470i).
func TestRunsTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{
{Name: Prefix + "_test_runs",
Description: "The pipeline runner's test runs, one per candidate, level and tested packages (the state test-runs): each with its packages read from its output object, its ask, node, readers, outcome and counts, and the state's fill against its cap setting test-runs-cap-mib; filtered by a candidate, a level, a digest or a module, or none.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"candidate": map[string]any{"type": "string"}, "level": map[string]any{"type": "string"},
"digest": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}}},
Run: func(args map[string]any) (any, error) {
in, err := as[seat.TestRunsInput](args)
if err != nil {
return nil, err
}
return ask("seat:"+seat.Seat+"."+seat.VerbTestRuns, in)
}},
{Name: Prefix + "_test_runs_drop",
Description: "Drop one test run by the seat verb test-runs-drop, a hand act recorded first: its ask is dropped, each reading step ended could not run naming the drop, and its key and output object deleted. It takes the run's key and why.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"key": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}, "required": []string{"key", "why"}},
Run: func(args map[string]any) (any, error) {
in, err := as[seat.TestRunsDropInput](args)
if err != nil {
return nil, err
}
if strings.TrimSpace(in.Key) == "" || strings.TrimSpace(in.Why) == "" {
return nil, errors.New("test-runs-drop names the run's key and why")
}
return ask("seat:"+seat.Seat+"."+seat.VerbTestRunsDrop, in)
}},
}
}
@@ -36,7 +36,7 @@ func deleteBound(ctx context.Context, bounds jetstream.KeyValue) func(key string
// armDropBound arms the five-minute bound of a drop, a key of the state bounds with its own time to live (A-77, A-222).
func armDropBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) error {
return func(key string) error {
_, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.AskBound))
_, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.DropBound))
if errors.Is(err, jetstream.ErrKeyExists) {
return nil
}
@@ -67,26 +67,19 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func
return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: id, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning,
Summary: fmt.Sprintf("the call %s of ask %s is unanswered", call, ask),
Headline: "A call whose answer is kept is not answered",
Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.AskBound),
Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.DropBound),
Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: id}})
}
}
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered, an ask's ends the step no
// holder took (§32, §535, A-77); another key is none of the runner's.
func boundExpired(h *pipeline.Hearer, unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none
// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them).
func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
return func(key string) {
if s, ask, ok := pipeline.ParseDropBound(key); ok {
if err := unanswered(s, ask); err != nil {
logf("the drop of ask %s on %s, unanswered, is not said: %v", ask, s, err)
}
return
}
if !pipeline.IsAskKey(key) {
return
}
if err := h.NotTaken(key, pipeline.AskBound); err != nil {
logf("the step of ask %s, which no holder took, is not ended: %v", key, err)
}
}
}
@@ -31,7 +31,7 @@ func (f *fakeFindings) Clear(_ context.Context, scope, id, kind, _ string) error
func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) {
w := &fakeFindings{}
var logged int
expired := boundExpired(nil, dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ })
expired := boundExpired(dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ })
expired("not-a-bound")
if len(w.written) != 0 {
t.Fatalf("a key of no drop was said: %+v", w.written)
@@ -67,8 +67,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// The bound of each ask a build seat's holder must take: a key with its own time to live, its expiry heard (§535, A-77).
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound an ask no holder takes")
// The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77).
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers")
if err != nil {
return err
}
@@ -151,14 +151,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// for its step.
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has}
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error {
n, err := starter.PlaceRun(ev, half, rv.Catalog.View)
logger.Printf("the candidate %s's revision %d placed %d pipelines of its run", ev.Group, ev.Revision, n)
return err
}
resume := func(branch string) {
rv.Holders.refresh() // a half held may name the claimant of the seat-holders view (§647)
half, ok := rv.Catalog.View(branch)
if !ok {
return
@@ -182,6 +183,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
resumer.Store(&resume)
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
advance := func(branch string) {
rv.Holders.refresh()
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
}
@@ -227,23 +229,41 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// A newer commit or publish supersedes the older running pipelines it replaces, their asks dropped as kept calls (§31,
// §350; A-182, A-202).
starter.Supersede = func(newer seat.Pipeline) {
n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)), dropBound(ctx, bounds))
n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)))
if err != nil {
logger.Printf("the pipelines %s supersedes: %d ended, then %v", newer.Pipeline, n, err)
} else if n > 0 {
logger.Printf("the pipeline %s superseded %d older pipelines", newer.Pipeline, n)
}
}
// The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock.
runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings))
runsKV, runObjs, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf)
if err != nil {
return err
}
if err := watchRunBounds(ctx, runsKV, hearer.RunExpired, logger.Printf); err != nil {
return err
}
if err := hearStepProgress(heard, hearer.RunProgress, logger.Printf); err != nil {
return err
}
if err := hearStepEnded(heard, hearer, logger.Printf); err != nil {
return err
}
if err := hearToolchain(ctx, heard, toolchains, logger.Printf); err != nil {
return err
}
if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil {
taken := func(ask, node string) error {
if err := hearer.Taken(ask, node); err != nil {
return err
}
return hearer.RunTaken(ask, node)
}
if err := hearAskTaken(heard, taken, logger.Printf); err != nil {
return err
}
expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
return err
}
@@ -261,6 +281,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// An entry whose step ended and was not appended is appended at the start, in order (§606, §549).
hearer.AppendUnlogged()
// A test run whose bound passed while the runner was away ends now, as its expiry would have ended it (§725).
if n, err := hearer.TakeUp(); err != nil || n > 0 {
logger.Printf("the test runs taken up at the start: %d ended past their bound (%v)", n, err)
}
go whileTheLogIsFull(ctx, fills, hearer, time.Minute)
// The seat list, read over the calls connection and heard on the hub (seatlist.Part), which the seat's write verbs check
// their callers against (A-186): registered before the hub starts, read again once its events are live.
@@ -273,7 +297,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24).
logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred))
// The seat's write verbs are answered here, by the process that holds the pipelines (A-107).
stopVerbs, err := answerVerbs(calls, seats, writeVerbs(ctx, hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls, bounds))
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
stopVerbs, err := answerVerbs(calls, seats, verbs)
if err != nil {
return err
}
@@ -65,8 +65,11 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
t.Fatal(err)
}
}
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
// bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl.
for _, state := range []string{"bounds", TestRunsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
}
}
// The step log, the runner's old logs section (§470): the stream the bus makes for it.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
@@ -114,7 +114,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
_ = m.Term()
return
}
_, err = h.OnStepEnded(bs, m.Subject(), ev)
var shared []seat.Shared
if bs.Shared != nil {
shared = bs.Shared(m.Data())
}
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
switch {
case errors.Is(err, pipeline.ErrRefused):
logf("%v", err)
@@ -133,11 +137,10 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
return nil
}
// hearAskTaken deletes the bound of each ask a build seat's holder took (§602, §647) and keeps on its step the node that took
// it (A-182): the ask is answered, so its five minutes end with no finding.
func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, taken func(ask, node string) error, logf func(string, ...any)) error {
// hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of
// that node's holder, and begins the clock of a test run's ask (§725).
func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error {
for _, ts := range pipeline.TakenSeats {
ts := ts
err := hub.Handle(ts.Subject, func(m jetstream.Msg) {
var ev nodebuildgo.AskTaken
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
@@ -145,12 +148,6 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue,
_ = m.Term()
return
}
if err := bounds.Delete(ctx, ev.Ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
logf("the bound of ask %s, taken on %s, is not deleted, heard again: %v", ev.Ask, ev.Node, err)
_ = m.Nak()
return
}
// The step keeps the node that took its ask, so a drop of it is asked of that node's holder (A-182).
if err := taken(ev.Ask, ev.Node); err != nil {
logf("the node %s that took ask %s is not kept on its step, heard again: %v", ev.Node, ev.Ask, err)
_ = m.Nak()
@@ -165,19 +162,8 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue,
return nil
}
// armBound is the bound of an ask: a key of the state bounds with its own time to live, named for the ask.
func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm {
return func(ask string) error {
_, err := bounds.Create(ctx, ask, []byte(`{"ask":"`+ask+`"}`), jetstream.KeyTTL(pipeline.AskBound))
if errors.Is(err, jetstream.ErrKeyExists) {
return nil
}
return err
}
}
// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32,
// §535, A-77), and says a drop no holder answered unanswered (A-222): no timer in the process.
// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered
// unanswered (A-222, A-77): no timer in the process.
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
@@ -0,0 +1,416 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// TestRunsState is the runner's state of its test runs (design 47 G §725), declared in its manifest's old state section.
const TestRunsState = "test-runs"
// testRunsKV is the state test-runs on the bus: its runs by compare-and-set, and its bound keys, each written with its own
// time to live (A-77, A-470).
type testRunsKV struct {
ctx context.Context
js jetstream.JetStream
kv jetstream.KeyValue
}
func (t testRunsKV) Create(key string, r pipeline.Run) error {
raw, err := json.Marshal(r)
if err != nil {
return err
}
if _, err := t.kv.Create(t.ctx, key, raw); errors.Is(err, jetstream.ErrKeyExists) {
return pipeline.ErrExists
} else if err != nil {
return err
}
return nil
}
func (t testRunsKV) Get(key string) (pipeline.Run, uint64, error) {
e, err := t.kv.Get(t.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return pipeline.Run{}, 0, pipeline.ErrNoRun
}
if err != nil {
return pipeline.Run{}, 0, err
}
var r pipeline.Run
if err := json.Unmarshal(e.Value(), &r); err != nil {
return pipeline.Run{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err)
}
return r, e.Revision(), nil
}
func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64, error) {
raw, err := json.Marshal(r)
if err != nil {
return 0, err
}
rev, err := t.kv.Update(t.ctx, key, raw, revision)
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
return 0, pipeline.ErrChanged
}
return rev, err
}
func (t testRunsKV) Delete(key string, revision uint64) error {
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
return pipeline.ErrChanged
}
if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
// deleteBound deletes a bound key, whose value is no run: a key gone is no fault.
func (t testRunsKV) deleteBound(key string) error {
if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
type boundValue struct {
Bound string `json:"bound"`
Ask string `json:"ask"`
}
// Bound writes key again holding ask, with ttl as its own time to live, published at the key's latest revision (a delete
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
// takes its time to live with it, so the expiry is heard only ttl after the last write.
func (t testRunsKV) Bound(key, ask string, ttl time.Duration) (uint64, error) {
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
if err != nil {
return 0, err
}
subject := "$KV." + t.kv.Bucket() + "." + key
for {
var last uint64
st, err := t.js.Stream(t.ctx, "KV_"+t.kv.Bucket())
if err != nil {
return 0, err
}
m, err := st.GetLastMsgForSubject(t.ctx, subject)
switch {
case err == nil:
last = m.Sequence
case !errors.Is(err, jetstream.ErrMsgNotFound):
return 0, err
}
ack, err := t.js.Publish(t.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
continue
}
if err != nil {
return 0, err
}
return ack.Sequence, nil
}
}
// Bounded reads a bound key: the ask it holds and its revision.
func (t testRunsKV) Bounded(key string) (string, uint64, bool, error) {
e, err := t.kv.Get(t.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return "", 0, false, nil
}
if err != nil {
return "", 0, false, err
}
var v boundValue
_ = json.Unmarshal(e.Value(), &v)
return v.Ask, e.Revision(), true, nil
}
func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) }
// UnboundAt deletes a bound key at revision: a later write of it is kept.
func (t testRunsKV) UnboundAt(key string, revision uint64) error {
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence") || errors.Is(err, jetstream.ErrKeyNotFound)) {
return nil
}
return err
}
func (t testRunsKV) Has(key string) (bool, error) {
_, err := t.kv.Get(t.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return false, nil
}
return err == nil, err
}
func (t testRunsKV) Keys() ([]string, error) {
all, err := t.kv.Keys(t.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return all, err
}
// runsKept is the state test-runs as the verb test-runs reads it (A-470i): its keys, and each key's value as written.
type runsKept struct {
ctx context.Context
kv jetstream.KeyValue
}
func (r runsKept) Keys() ([]string, error) {
ks, err := r.kv.Keys(r.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return ks, err
}
func (r runsKept) Value(key string) (json.RawMessage, error) {
e, err := r.kv.Get(r.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return e.Value(), nil
}
// runObjects is the state test-runs' own object store, `test-runs-objects`, made with the state (A-469e), opened when
// first used.
type runObjects struct {
ctx context.Context
js jetstream.JetStream
module string
mu sync.Mutex
store jetstream.ObjectStore
}
func (o *runObjects) open() (jetstream.ObjectStore, error) {
o.mu.Lock()
defer o.mu.Unlock()
if o.store != nil {
return o.store, nil
}
s, err := o.js.ObjectStore(o.ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(TestRunsState)))
if err != nil {
return nil, fmt.Errorf("the object store of the state %s is not open: %w", TestRunsState, err)
}
o.store = s
return s, nil
}
func (o *runObjects) Put(name string, data []byte) error {
s, err := o.open()
if err != nil {
return err
}
_, err = s.PutBytes(o.ctx, name, data)
return err
}
func (o *runObjects) Get(name string) ([]byte, error) {
s, err := o.open()
if err != nil {
return nil, err
}
data, err := s.GetBytes(o.ctx, name)
if errors.Is(err, jetstream.ErrObjectNotFound) {
return nil, pipeline.ErrNoRun
}
return data, err
}
func (o *runObjects) Delete(name string) error {
s, err := o.open()
if err != nil {
return err
}
if err := s.Delete(o.ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
return err
}
return nil
}
// runsFill reads test-runs' fill, its key-value stream's bytes (A-469d): judged by the process against its cap setting.
func runsFill(ctx context.Context, js jetstream.JetStream, module string, now func() time.Time) func() (seat.Fill, error) {
return func() (seat.Fill, error) {
st, err := js.Stream(ctx, "KV_"+ownBucket(module, TestRunsState))
if err != nil {
return seat.Fill{}, err
}
info, err := st.Info(ctx)
if err != nil {
return seat.Fill{}, err
}
at := now().UTC()
return seat.Fill{Store: TestRunsState, Bytes: int64(info.State.Bytes), Cap: info.Config.MaxBytes, At: &at}, nil
}
}
// The files the manifest writes each setting of the test runs into, `${config:<key>}`, named by the environment (A-469x).
const (
envRunsCap = "MESH_RUNNER_TEST_RUNS_CAP_FILE"
envRunsKept = "MESH_RUNNER_TEST_RUNS_KEPT_FILE"
envStepBound = "MESH_RUNNER_TEST_STEP_BOUND_FILE"
envUntakenBound = "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE"
)
// maxRunsKept is test-runs-kept's maximum (A-470a): the pipeline naming a run's key is kept at least that long.
const maxRunsKept = 168 * time.Hour
// runSettings reads the settings of the test runs at the start (§725; A-469x): test-runs-cap-mib a whole number of MiB,
// test-runs-kept, test-step-bound and run-untaken-bound Go durations. A setting whose file is missing or empty is its
// default; a content that is neither is an error naming the file.
func runSettings(getenv func(string) string) (pipeline.RunSettings, error) {
s := pipeline.DefaultRunSettings
if v, err := settingText(getenv(envRunsCap)); err != nil {
return s, err
} else if v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return s, fmt.Errorf("%s holds %q, which is not a positive whole number", getenv(envRunsCap), v)
}
s.CapBytes = int64(n) << 20
}
for _, d := range []struct {
env string
to *time.Duration
}{{envRunsKept, &s.Kept}, {envStepBound, &s.StepBound}, {envUntakenBound, &s.Untaken}} {
v, err := settingText(getenv(d.env))
if err != nil {
return s, err
}
if v == "" {
continue
}
t, err := time.ParseDuration(v)
if err != nil || t <= 0 {
return s, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(d.env), v)
}
*d.to = t
}
if s.Kept > maxRunsKept {
return s, fmt.Errorf("%s holds %s, past test-runs-kept's maximum %s (A-470a)", getenv(envRunsKept), s.Kept, maxRunsKept)
}
return s, nil
}
// settingText is one setting file's content, trimmed: empty for no path, no file or an empty one.
func settingText(path string) (string, error) {
if path == "" {
return "", nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return "", nil
}
if err != nil {
return "", err
}
return strings.TrimSpace(string(raw)), nil
}
// progressSubjects hear every build seat's step-progress (§725), from their contracts.
var progressSubjects = []string{nodebuildgo.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll}
// hearStepProgress begins again the clock of the test run whose ask a build seat's holder says progress of (§725, A-466p).
func hearStepProgress(hub handlers, progress func(ask string) error, logf func(string, ...any)) error {
for _, subject := range progressSubjects {
err := hub.Handle(subject, func(m jetstream.Msg) {
var ev nodebuildgo.StepProgress
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
logf("%s is no step-progress: %v", m.Subject(), err)
_ = m.Term()
return
}
if err := progress(ev.Ask); err != nil {
logf("the clock of the test run of ask %s is not begun again, heard again: %v", ev.Ask, err)
_ = m.Nak()
return
}
_ = m.Ack()
})
if err != nil {
return err
}
}
return nil
}
// watchRunBounds hears the expiry of each test run's bound key, the bus's marker of the key's end (§725): no timer in the
// process.
func watchRunBounds(ctx context.Context, kv jetstream.KeyValue, expired func(key string) error, logf func(string, ...any)) error {
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
return err
}
go func() {
for {
select {
case <-ctx.Done():
_ = w.Stop()
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsRunBound(e.Key()) {
continue
}
if err := expired(e.Key()); err != nil {
logf("the test run of the bound %s that ended is not ended: %v", e.Key(), err)
}
}
}
}()
return nil
}
// openTestRuns opens the state test-runs and wires it into the starter, the hearer and the ager (§725).
func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, getenv func(string) string,
starter *pipeline.Starter, hearer *pipeline.Hearer, drop pipeline.Drop, logf func(string, ...any)) (jetstream.KeyValue, *runObjects, error) {
settings, err := runSettings(getenv)
if err != nil {
return nil, nil, err
}
kv, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, TestRunsState), "share a test run")
if err != nil {
return nil, nil, err
}
runs := testRunsKV{ctx: ctx, js: js, kv: kv}
objects := &runObjects{ctx: ctx, js: js, module: cred.Module}
starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings
starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now)
starter.EndReaders = hearer.EndReaders
hearer.TestRuns, hearer.Clock, hearer.Runs, hearer.DropRun = runs, runs, settings, drop
hearer.RunOutputs = objects
if hearer.Ager != nil {
hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept = runs, objects.Delete, settings.Kept
}
logf("test runs: cap %d bytes, kept %s, step bound %s, untaken bound %s", settings.CapBytes, settings.Kept, settings.StepBound, settings.Untaken)
return kv, objects, nil
}
@@ -0,0 +1,579 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/testbus"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
"mesh-delivery-pipeline/internal/pipeline"
)
// Test 109 (design 47 G §733 *Whole (runner…)*, §724 to §728, §535 and §647 as A-466y and A-466z amend them; A-468f: in
// mesh-delivery with a fixture build seat holder; A-470f: no reader-only refusal). The runner whole against the real bus: a
// module whose program and tools bundle test one directory asks one unit and one integration run, both artifacts' steps
// ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step
// that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a
// run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed;
// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended,
// and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the
// same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its
// other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and
// test-runs-drop deletes the key and its output object (A-470i); at the cap a step ends could not run, naming the fill.
// The step bound is two seconds here, as the setting test-step-bound names it, where the mesh's is 30 minutes.
func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
}
for _, state := range []string{"bounds", TestRunsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
}
}
// test-runs' own object store, made with the state (A-469e).
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
t.Fatal(err)
}
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}})
if err != nil {
t.Fatal(err)
}
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
// A durable consumer of the runner's step-ended in EVENTS, so an end said again is read as the stream keeps it (A-470q).
stepEnds, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "t109-step-ended",
FilterSubject: "mesh.seat." + seat.Seat + ".event." + seat.EventStepEnded + ".>", AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
t.Fatal(err)
}
// The seat-holders view of the seat mesh-module-assigner's claimant (§647, A-466z), served as its owner does: node-build-go
// has a holder, node-build-node none.
holders := viewstest.Serve(t, ctx, js, nc, "assigner", "seat-holders", "assigner.seat-holders", "n1_"+module)
if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil {
t.Fatal(err)
}
// The registrations: app tests its program and tools bundle from one directory, its unit step required and its
// integration step optional; web is built by node-build-node.
policy := catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "optional", Composition: "bypass", Review: "bypass", Approval: "bypass"}}
regs := []catalog.Registration{{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: policy},
{Module: "web", Branches: []string{"main"}, Origin: "o/web", Policy: policy}}
answer := func(subject string, body any) {
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": body})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
}
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs})
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
entries := []catalog.CatalogEntry{
{Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app",
Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
{Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}},
{Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
{Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "node"}}},
{Module: "assigner", Manifest: map[string]any{"module": "assigner", "version": "0.1", "claims": []any{map[string]any{"name": seatlist.HolderSeat, "scope": "mesh"}}}},
}
var revision atomic.Int64
revision.Store(1)
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) {
r := revision.Load()
raw, _ := json.Marshal(map[string]any{"result": catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: r, Kind: "trunk", Branch: "main", Source: 2 + r}, Entries: entries}})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json")
if err != nil {
t.Fatal(err)
}
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) {
var in struct {
Seat string `json:"seat"`
}
_ = json.Unmarshal(m.Data, &in)
entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"}
var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}}
if in.Seat != "" {
entry.Definition = def
result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1}
}
raw, _ := json.Marshal(map[string]any{"result": result})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
// Every pipeline-ended said, and every drop asked of a build seat.
var mu sync.Mutex
var endings []seat.PipelineEnded
var drops []string
if _, err := nc.Subscribe("mesh.seat.mesh-pipeline-runner.event.pipeline-ended.>", func(m *nats.Msg) {
var e seat.PipelineEnded
if json.Unmarshal(m.Data, &e) == nil {
mu.Lock()
endings = append(endings, e)
mu.Unlock()
}
}); err != nil {
t.Fatal(err)
}
if _, err := nc.Subscribe("mesh.seat.*.call.drop.>", func(m *nats.Msg) {
mu.Lock()
drops = append(drops, string(m.Data))
mu.Unlock()
raw, _ := json.Marshal(map[string]any{"result": map[string]any{}})
_ = m.Respond(raw)
}); err != nil {
t.Fatal(err)
}
ended := func(key string) []seat.PipelineEnded {
mu.Lock()
defer mu.Unlock()
var out []seat.PipelineEnded
for _, e := range endings {
if e.Pipeline == key {
out = append(out, e)
}
}
return out
}
// The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds.
dir := t.TempDir()
files := map[string]string{envRunsCap: "1", envStepBound: "2s"}
env := map[string]string{}
for name, content := range files {
path := filepath.Join(dir, name)
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
env[name] = path
}
broker := filepath.Join(dir, "broker")
raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"})
if err := os.WriteFile(broker, raw, 0o600); err != nil {
t.Fatal(err)
}
env[process.BrokerFileEnv] = broker
logs := &syncBuffer{}
start := func() func() {
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }()
return func() { stop(); <-done }
}
stop := start()
defer func() { stop() }()
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
if err != nil {
t.Fatal(err)
}
runsKV, err := js.KeyValue(ctx, module+"_"+TestRunsState)
if err != nil {
t.Fatal(err)
}
firstViews, err := js.KeyValue(ctx, module+"_"+views.StatusState)
if err != nil {
t.Fatal(err)
}
// The seat-holders view's first view is read before any pipeline is placed, so a seat is judged from it (A-466y).
waitFor(t, ctx, logs, func() bool {
e, err := firstViews.Get(ctx, views.StatusKey("assigner", "seat-holders"))
var s views.Status
return err == nil && json.Unmarshal(e.Value(), &s) == nil && !s.Answered.IsZero()
})
takeIn := func(module, commit, fp string, source, rev int64) {
taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m." + commit, Kind: "taken", Branch: "main", Source: source, Revision: rev,
Entries: []catalog.TakeInEntry{{Module: module, Commit: commit, Fingerprint: fp}}})
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil {
t.Fatal(err)
}
}
placed := func(key string) seat.Pipeline {
var p seat.Pipeline
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && len(p.Steps) > 1 })
return p
}
read := func(key string) seat.Pipeline {
var p seat.Pipeline
readPipeline(ctx, pipelines, key, &p)
return p
}
step := func(p seat.Pipeline, artifact, name string) seat.StepResult {
for _, s := range p.Steps {
if s.Artifact == artifact && s.Step == name {
return s
}
}
return seat.StepResult{}
}
runOf := func(key string) (pipeline.Run, bool) {
e, err := runsKV.Get(ctx, key)
var r pipeline.Run
return r, err == nil && json.Unmarshal(e.Value(), &r) == nil
}
runKeys := func(candidate string) []string {
ks, _ := runsKV.Keys(ctx)
var out []string
for _, k := range ks {
if !pipeline.IsRunBound(k) && strings.HasPrefix(k, candidate+".") {
out = append(out, k)
}
}
return out
}
runOfStep := func(key, artifact, name string) (string, pipeline.Run) {
var run string
var r pipeline.Run
waitFor(t, ctx, logs, func() bool {
s := step(read(key), artifact, name)
if len(s.Shared) == 0 {
return false
}
run = s.Shared[0].Run
got, ok := runOf(run)
r = got
return ok && r.Ask != ""
})
return run, r
}
publish := func(subject string, body any) {
raw, _ := json.Marshal(body)
if _, err := js.Publish(ctx, subject, raw); err != nil {
t.Fatal(err)
}
}
taken := func(ask string) {
publish(nodebuildgo.SubjectAskTaken("n1"), nodebuildgo.AskTaken{Ask: ask, Asker: module, Node: "n1", At: time.Now()})
}
endAsk := func(key, ask, name, outcome string, tests *nodebuildgo.TestCounts) {
now := time.Now()
publish(nodebuildgo.SubjectStepEnded("n1"), nodebuildgo.StepEnded{ID: ask, Ask: ask, Asker: module, Pipeline: key, Module: "app", Commit: "c1",
Fingerprint: "src2:aa", Step: name, Node: "n1", Outcome: outcome, Tests: tests, At: now, Ended: &now})
}
passBuild := func(key, module, commit, fp string) {
ask := step(read(key), "", seat.StepResultStepBuild).Ask
publish("mesh.seat.mesh-builds.event.step-ended.n1", builds.StepEnded{ID: ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed",
Entries: []builds.BuildEntry{{Module: module, Commit: commit, Fingerprint: fp}}})
}
call := func(verb string, in, out any) string {
var reply struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
waitFor(t, ctx, logs, func() bool {
m, err := nc.Request("mesh.seat.mesh-pipeline-runner.call."+verb+".agent~dev", mustJSON(in), 2*time.Second)
reply.Error = ""
return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read")
})
if out != nil && reply.Error == "" {
_ = json.Unmarshal(reply.Result, out)
}
return reply.Error
}
// One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its
// ask is taken (A-469b), its holder present and nothing failing it (A-466y).
const p3 = "main.3.app"
takeIn("app", "c1", "src2:aa", 3, 1)
passBuild(p3, "app", "c1", "src2:aa")
unitRun, unit := runOfStep(p3, "app:program", "unit")
integRun, integ := runOfStep(p3, "app:program", "integration")
if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 {
t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers)
}
if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask {
t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s)
}
// Its wait has no bound (§647 as A-466y amends it): the state bounds holds no key for a queued ask, only drops' keys.
boundsKV, err := js.KeyValue(ctx, module+"_bounds")
if err != nil {
t.Fatal(err)
}
if ks, err := boundsKV.Keys(ctx); err != nil && err != jetstream.ErrNoKeysFound {
t.Fatal(err)
} else {
for _, k := range ks {
if !strings.HasPrefix(k, pipeline.DropBoundPrefix) {
t.Errorf("the state bounds holds %s after the asks were put; want no bound of a queued ask", k)
}
}
}
// A runner restarted during a run takes it up again by its ask (§725): the run's end, heard after the restart, ends both.
taken(unit.Ask)
waitFor(t, ctx, logs, func() bool { r, _ := runOf(unitRun); return r.Node == "n1" })
// Its clock holds the run's ask (A-470n).
waitFor(t, ctx, logs, func() bool {
e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun)
return err == nil && strings.Contains(string(e.Value()), `"ask":"`+unit.Ask+`"`)
})
stop()
stop = start()
// It keeps saying its progress past its bound and is not ended (§725, §720).
// Each progress is heard before the next is said: the run's clock is written again (its bound key's revision rises).
boundRev := func() uint64 {
e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun)
if err != nil {
return 0
}
return e.Revision()
}
for at := time.Now(); time.Since(at) < 3*time.Second; {
before := boundRev()
publish(nodebuildgo.SubjectStepProgress("n1"), nodebuildgo.StepProgress{Ask: unit.Ask, Asker: module, Phase: "run", At: time.Now()})
waitFor(t, ctx, logs, func() bool { return boundRev() > before })
}
if r, _ := runOf(unitRun); r.Ended != nil {
t.Fatalf("a run saying its progress past its bound ended: %+v", r)
}
// A failing test fails once and is counted once (§726).
endAsk(p3, unit.Ask, "unit", "failed", &nodebuildgo.TestCounts{Unit: 5})
taken(integ.Ask)
endAsk(p3, integ.Ask, "integration", "failed", &nodebuildgo.TestCounts{Integration: 2})
waitFor(t, ctx, logs, func() bool { return len(ended(p3)) == 1 })
p := read(p3)
for _, a := range []string{"app:program", "app:tools"} {
if s := step(p, a, "unit"); s.Outcome != seat.StepResultOutcomeFailed || len(s.Shared) == 0 || s.Shared[0].Run != unitRun {
t.Errorf("%s's unit step = %+v; want failed with the run in shared", a, s)
}
}
for _, k := range []string{unitRun, integRun} {
if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomeFailed {
t.Errorf("the run %s at its end = %+v; want it ended failed", k, r)
}
}
if e := ended(p3)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Tests == nil || e.Tests.Unit != 5 || e.Tests.Integration != 2 {
t.Fatalf("pipeline-ended = %+v; want failed, the tests counted once", e)
}
// An accepted step does not move; a rerun of the program's unit step that passes moves the tools' to passed, and the
// pipeline says pipeline-ended again (§727, A-470d).
if e := call(seat.VerbAccept, seat.AcceptInput{Pipeline: p3, Step: "integration", Artifact: "app:program", Why: "flaky"}, nil); e != "" {
t.Fatalf("accept: %s", e)
}
if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "unit", Artifact: "app:program", Why: "fixed"}, nil); e != "" {
t.Fatalf("rerun: %s", e)
}
var again pipeline.Run
waitFor(t, ctx, logs, func() bool { r, ok := runOf(unitRun); again = r; return ok && r.Ended == nil && r.Ask != unit.Ask })
taken(again.Ask)
endAsk(p3, again.Ask, "unit", "passed", &nodebuildgo.TestCounts{Unit: 5})
if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "integration", Artifact: "app:tools", Why: "fixed"}, nil); e != "" {
t.Fatalf("rerun: %s", e)
}
var integAgain pipeline.Run
waitFor(t, ctx, logs, func() bool {
r, ok := runOf(integRun)
integAgain = r
return ok && r.Ended == nil && r.Ask != integ.Ask
})
taken(integAgain.Ask)
endAsk(p3, integAgain.Ask, "integration", "passed", nil)
waitFor(t, ctx, logs, func() bool {
e := ended(p3)
return len(e) >= 2 && e[len(e)-1].Outcome == seat.PipelineEndedOutcomePassed
})
p = read(p3)
if s := step(p, "app:tools", "unit"); s.Outcome != seat.StepResultOutcomePassed || !s.Rerun {
t.Errorf("the tools' unit step = %+v; want moved to passed, marked a rerun", s)
}
// The tools' unit step's ends, read from the stream: its first, failed, and its end said again by the move, passed.
var toolEnds []string
for {
batch, err := stepEnds.FetchNoWait(100)
if err != nil {
t.Fatal(err)
}
n := 0
for m := range batch.Messages() {
n++
var e seat.StepEvent
if json.Unmarshal(m.Data(), &e) == nil && e.Pipeline == p3 && e.Result.Artifact == "app:tools" && e.Result.Step == "unit" {
toolEnds = append(toolEnds, e.Result.Outcome)
}
_ = m.Ack()
}
if n == 0 {
break
}
}
if len(toolEnds) < 2 || toolEnds[0] != seat.StepResultOutcomeFailed || toolEnds[len(toolEnds)-1] != seat.StepResultOutcomePassed {
t.Errorf("the tools' unit step-ended in EVENTS = %v; want its first end failed and the end said again passed, both kept", toolEnds)
}
if s := step(p, "app:program", "integration"); s.Outcome != seat.StepResultOutcomeAccepted {
t.Errorf("the accepted step = %+v; want it accepted still", s)
}
for _, k := range []string{unitRun, integRun} {
if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomePassed {
t.Errorf("the run %s after its rerun = %+v; want it ended passed", k, r)
}
}
if e := ended(p3); e[len(e)-1].EndCount <= e[0].EndCount {
t.Errorf("pipeline-ended counts %d then %d; want a higher end count", e[0].EndCount, e[len(e)-1].EndCount)
}
// A second revision with the same digests asks its own runs, so a failed test runs again (§724).
revision.Store(2)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4})
const p4, p5 = "main.4.app", "main.5.app"
takeIn("app", "c1", "src2:aa", 4, 2)
placed(p4)
run4, r4 := runOfStep(p4, "app:program", "integration")
if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask {
t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask)
}
// A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads
// the run, the first is cancelled and leaves it, and the run's ask is not dropped.
takeIn("app", "c1", "src2:aa", 5, 2)
placed(p5)
waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 })
if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" {
t.Fatalf("cancel: %s", e)
}
waitFor(t, ctx, logs, func() bool {
r, _ := runOf(run4)
return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5
})
if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask {
t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r)
}
mu.Lock()
dropped := append([]string(nil), drops...)
mu.Unlock()
if len(dropped) != 0 {
t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String())
}
// A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720).
taken(r4.Ask)
waitFor(t, ctx, logs, func() bool {
return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun &&
step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun
})
if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") {
t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r)
}
waitFor(t, ctx, logs, func() bool {
mu.Lock()
defer mu.Unlock()
return len(drops) == 1 && strings.Contains(drops[0], r4.Ask)
})
// A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not
// asked (§535, §647; A-470g, A-470h).
const web = "main.5.web"
takeIn("web", "w1", "src2:ww", 5, 2)
waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 })
if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" {
t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e)
}
// The steps of the seat with no holder end could not run, a bypassed step stays bypassed, every other ends not asked,
// and none is asked (A-470h).
for _, s := range read(web).Steps {
want := seat.StepResultOutcomeNotAsked
switch {
case s.Status == seat.StepResultStatusBypass:
want = seat.StepResultOutcomeBypassed
case s.Seat == "node-build-node":
want = seat.StepResultOutcomeCouldNotRun
}
if s.Outcome != want || s.Ask != "" {
t.Errorf("the step %s %s of the pipeline with no holder = %s (ask %q); want %s", s.Artifact, s.Step, s.Outcome, s.Ask, want)
}
}
// test-runs is answered by the process, its fill against test-runs-cap-mib (A-470i), each run's packages from its output
// object; test-runs-drop deletes the key and its object (§728).
var runs seat.TestRunsAnswer
if e := call(seat.VerbTestRuns, seat.TestRunsInput{Candidate: "main.1"}, &runs); e != "" || len(runs.Runs) != 2 || runs.Fill.Cap != 1<<20 || runs.Fill.Store != TestRunsState {
t.Fatalf("test-runs = %+v (%s); want main.1's two runs, the fill against 1 MiB", runs, e)
}
objects, err := js.ObjectStore(ctx, catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState)))
if err != nil {
t.Fatal(err)
}
if _, err := objects.GetInfo(ctx, unitRun); err != nil {
t.Fatalf("the unit run's output object: %v", err)
}
var drop seat.TestRunsDropAnswer
if e := call(seat.VerbTestRunsDrop, seat.TestRunsDropInput{Key: unitRun, Why: "test 109"}, &drop); e != "" || drop.Run.Key != unitRun {
t.Fatalf("test-runs-drop = %+v (%s)", drop, e)
}
if _, ok := runOf(unitRun); ok {
t.Error("the dropped run's key is kept")
}
if _, err := objects.GetInfo(ctx, unitRun); err == nil {
t.Error("the dropped run's output object is kept")
}
// At the cap a step ends could not run, naming the fill (§725, A-469d).
pad := make([]byte, 600<<10)
for _, k := range []string{"pad.a", "pad.b"} {
if _, err := runsKV.Put(ctx, k, pad); err != nil {
t.Fatal(err)
}
}
revision.Store(3)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5})
const p6 = "main.6.app"
takeIn("app", "c1", "src2:aa", 6, 3)
waitFor(t, ctx, logs, func() bool {
s := step(read(p6), "app:program", "unit")
return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib")
})
}
func mustJSON(v any) []byte {
raw, _ := json.Marshal(v)
return raw
}
@@ -0,0 +1,58 @@
package main
import (
"os"
"path/filepath"
"testing"
"time"
)
// The test runs' settings reach the process through the files the manifest writes (design 47 G §725; A-469x, A-470a): the
// cap a whole number of MiB, the times Go durations, a missing file its default, a content that is neither, or a kept time
// past 168h, an error naming the file.
func TestTheTestRunsSettingsAreReadFromTheirFiles(t *testing.T) {
dir := t.TempDir()
write := func(name, content string) string {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return p
}
rows := []struct {
name string
files map[string]string
ok bool
check func(t *testing.T, cap int64, kept, step, untaken time.Duration)
}{
{"defaults", nil, true, func(t *testing.T, cap int64, kept, step, untaken time.Duration) {
if cap != 16<<20 || kept != 168*time.Hour || step != 30*time.Minute || untaken != 24*time.Hour {
t.Errorf("defaults = %d %s %s %s", cap, kept, step, untaken)
}
}},
{"set", map[string]string{envRunsCap: "8\n", envRunsKept: "48h", envStepBound: "2s", envUntakenBound: "1m"}, true,
func(t *testing.T, cap int64, kept, step, untaken time.Duration) {
if cap != 8<<20 || kept != 48*time.Hour || step != 2*time.Second || untaken != time.Minute {
t.Errorf("settings = %d %s %s %s", cap, kept, step, untaken)
}
}},
{"a cap that is no whole number", map[string]string{envRunsCap: "8.5"}, false, nil},
{"a time that is no duration", map[string]string{envStepBound: "30"}, false, nil},
{"kept past its maximum", map[string]string{envRunsKept: "169h"}, false, nil},
}
for _, r := range rows {
t.Run(r.name, func(t *testing.T) {
env := map[string]string{}
for k, v := range r.files {
env[k] = write(r.name+k, v)
}
s, err := runSettings(func(k string) string { return env[k] })
if (err == nil) != r.ok {
t.Fatalf("runSettings = %+v, %v", s, err)
}
if r.check != nil {
r.check(t, s.CapBytes, s.Kept, s.StepBound, s.Untaken)
}
})
}
}
@@ -1,13 +1,10 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
@@ -41,7 +38,8 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu
return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v))
}
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again.
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again; and
// the test runs' verbs test-runs and test-runs-drop (A-469g, A-470i).
func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }),
@@ -51,24 +49,17 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }),
seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }),
}
}
// dropBound deletes the bound of an ask a cancelled pipeline no longer waits on.
func dropBound(ctx context.Context, bounds jetstream.KeyValue) func(string) error {
return func(ask string) error {
if err := bounds.Delete(ctx, ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
// §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs.
seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }),
// A-470i: test-runs too, since the process holds test-runs-cap-mib, the cap its fill is said against.
seat.VerbTestRuns: verbOf(func(_ string, in seat.TestRunsInput) (any, error) { return v.Runs.TestRuns(in) }),
}
}
// writeVerbs are the verbs over the hearer and the starter, their hand acts recorded through the seat mesh-hand-acts.
func writeVerbs(ctx context.Context, h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool),
calls *process.Conn, bounds jetstream.KeyValue) pipeline.Verbs {
func writeVerbs(h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool), calls *process.Conn) pipeline.Verbs {
candidate := func(group, branch string, revision int64) (pipeline.Half, error) {
return readCandidate(calls.CallKept, catalog.CatalogRevisionEvent{Kind: "candidate", Group: group, Branch: branch, Revision: revision})
}
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}, Drop: dropBound(ctx, bounds)}
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}}
}
@@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) {
got = append(got, v)
}
slices.Sort(got)
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun}
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop}
slices.Sort(want)
if !slices.Equal(got, want) {
t.Errorf("the process answers %v, want %v", got, want)
@@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
"sync"
"sync/atomic"
"time"
@@ -12,6 +13,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-pipeline/internal/pipeline"
@@ -26,6 +28,70 @@ type runnerViews struct {
// Findings is the runner's held finding writer: it raises the finding pipeline-failed (§33) and says the runner's own
// findings (A-206).
Findings *findingsWriter
// Holders follows the seat-holders view of the seat mesh-module-assigner's claimant (§535, §647; A-466z).
Holders *holderView
}
// holderView follows the view seat-holders of the module the catalog halves name as the claimant of the seat
// mesh-module-assigner (seatlist.HolderSeat), as the runner's manifest consumes it, `seat:mesh-module-assigner.views.seat-holders`
// (design 47 G §535, §647 as A-466z amends them; §717): a claimant named anew drops the view followed and follows the new one's.
// Nothing is asked on a timer: it is read again whenever a branch's half is held or advances.
type holderView struct {
reader *views.ProcessReader
catalog *pipeline.Views
logf func(string, ...any)
mu sync.Mutex
owner string
pic *views.Picture
}
// refresh follows the claimant's seat-holders view when the claimant is another than the one followed.
func (h *holderView) refresh() {
var halves []pipeline.Half
for _, b := range h.catalog.Branches() {
if half, ok := h.catalog.View(b); ok {
halves = append(halves, half)
}
}
claimant, ok := pipeline.ClaimantOf(halves, seatlist.HolderSeat)
h.mu.Lock()
defer h.mu.Unlock()
if !ok || claimant == h.owner {
return
}
if h.owner != "" {
if err := h.reader.Drop(h.owner, seatHoldersState); err != nil {
h.logf("the seat-holders view of %s is not dropped: %v", h.owner, err)
}
h.owner, h.pic = "", nil
}
pic, err := h.reader.Follow(claimant, seatHoldersState, claimant+"."+seatHoldersState)
if err != nil {
h.logf("the seat-holders view of %s is not followed: %v", claimant, err)
return
}
h.owner, h.pic = claimant, pic
h.logf("the seat-holders view of %s, the claimant of %s, is followed", claimant, seatlist.HolderSeat)
}
// seatHoldersState is the seat mesh-module-assigner's view of who holds each seat (§616, §699).
const seatHoldersState = "seat-holders"
// Holders are the modules the followed view names for seatName, and whether its first view is read: before it, and with
// no claimant followed, nothing is judged (A-466y).
func (h *holderView) Holders(seatName string) ([]string, bool) {
h.mu.Lock()
pic := h.pic
h.mu.Unlock()
if pic == nil {
return nil, false
}
select {
case <-pic.Follower().Ready():
default:
return nil, false
}
return seatlist.HoldersIn(func() (map[string]json.RawMessage, error) { return pic.Follower().Values(), nil })(seatName), true
}
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
@@ -88,7 +154,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
return nil, err
}
reader.Start(ctx, js, cred, status)
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings}, nil
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil
}
// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it.
+2
View File
@@ -9,6 +9,8 @@ require (
)
require (
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
@@ -1,6 +1,7 @@
package pipeline
import (
"errors"
"fmt"
"time"
@@ -53,7 +54,12 @@ type Ager struct {
Get(key string) (seat.KeptResult, bool, error)
Delete(key string) error
}
Now func() time.Time
// Runs is the state test-runs, RunOutputs deletes an object of test-runs-objects and RunsKept is the setting
// test-runs-kept: an ended run is deleted with its output object once RunsKept has passed since its end (§725, A-470a).
Runs TestRuns
RunOutputs func(name string) error
RunsKept time.Duration
Now func() time.Time
// Logf is the runner's journal: each delete writes a line naming the key or object and its retention.
Logf func(string, ...any)
}
@@ -69,7 +75,16 @@ func (a *Ager) Age() {
if a == nil || a.Log == nil {
return
}
for _, r := range []Retention{Retention7, Retention30, Retention90} {
retentions := []Retention{Retention7, Retention30, Retention90}
if a.Runs != nil {
// The test runs age first, in the same pass, so the pipeline that names a run's key is still kept (A-470a).
kept := a.RunsKept
if kept <= 0 {
kept = DefaultRunSettings.Kept
}
retentions = append([]Retention{{RetentionTestRuns, kept}}, retentions...)
}
for _, r := range retentions {
if err := a.ageAt(r); err != nil {
a.logf("aged: the data past %s are not aged: %v", r.Name, err)
}
@@ -103,8 +118,57 @@ func (a *Ager) ageAt(r Retention) error {
return a.Aged.Put(r.Name, moved)
}
// RetentionTestRuns is the name, in the state aged, of the test runs' retention, the setting test-runs-kept (§725).
const RetentionTestRuns = "test-runs"
// ageRun deletes the test run the ended step of e read, and its output object, once its kept time has passed since the
// step's end, in the step log's order (§725, A-470a): the run's key is read from e's pipeline, its step's Shared run.
func (a *Ager) ageRun(r Retention, e seat.LogEntry) {
if e.What != seat.LogEntryWhatEnded {
return
}
p, _, err := a.Pipelines.Get(e.Pipeline)
if err != nil {
return
}
run := ""
for _, s := range p.Steps {
if s.Step == e.Step && s.Artifact == e.Artifact {
run = sharedRun(s)
}
}
if run == "" {
return
}
for {
t, rev, err := a.Runs.Get(run)
// The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not
// ended yet is aged at a later reader's end.
if err != nil || t.Ended == nil {
return
}
if t.Output != "" && a.RunOutputs != nil {
if err := a.RunOutputs(t.Output); err != nil {
a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err)
return
}
}
// Deleted at the revision read: a rerun that wrote the key again since is read again, and a run it started is kept.
if err := a.Runs.Delete(run, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
a.logf("aged: the test run %s is not deleted: %v", run, err)
return
}
a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age)
return
}
}
func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) {
switch r.Name {
case RetentionTestRuns:
a.ageRun(r, e)
case Retention7.Name:
if e.Output != "" && a.Outputs != nil {
if err := a.Outputs(e.Output); err != nil {
@@ -38,7 +38,7 @@ func approvalDue(p seat.Pipeline) int {
}
continue
}
if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning {
if s.Outcome == seat.StepResultOutcomeWaiting || open(s) {
return -1
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
@@ -1,20 +1,19 @@
package pipeline
import (
"errors"
"fmt"
"strings"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
)
// AskBound is how long an ask on a build seat's work queue may wait for its holder to take it (design 47 G §535, §647): five
// minutes, the bound of a key of the state `bounds` with its own time to live (A-77, A-104), never a timer.
const AskBound = 5 * time.Minute
// DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes,
// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as
// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725).
const DropBound = 5 * time.Minute
// TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it.
type TakenSeat struct {
@@ -23,15 +22,15 @@ type TakenSeat struct {
Subject string
}
// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken,
// so no bound is armed for their asks.
// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken.
var TakenSeats = []TakenSeat{
{Seat: nodebuildgo.Seat, Subject: nodebuildgo.SubjectAskTakenAll},
{Seat: nodebuildnode.Seat, Subject: nodebuildnode.SubjectAskTakenAll},
{Seat: nodebuildimage.Seat, Subject: nodebuildimage.SubjectAskTakenAll},
}
// BoundSeat says whether the holder of a seat says ask-taken, so an ask put on its work queue has a bound.
// BoundSeat says whether the holder of a seat says ask-taken: a build seat, whose asks are dropped and whose test steps share
// a run (§724).
func BoundSeat(name string) bool {
for _, s := range TakenSeats {
if s.Seat == name {
@@ -41,63 +40,6 @@ func BoundSeat(name string) bool {
return false
}
// Arm puts a bound of one ask's wait for its holder: a key of the state `bounds` named for the ask, written with its own
// time to live. It is armed before the ask is put, so an ask-taken heard at once finds it.
type Arm func(ask string) error
// NotTaken ends the step asked under ask that no holder took within its bound, the bound's expiry heard (design 47 G §32,
// §535, §647, A-77): the step ends `could not run` naming the seat, so its pipeline ends failed once the others end, never
// a silent wait. A step that ended or was never asked of a seat takes nothing.
func (h *Hearer) NotTaken(ask string, bound time.Duration) error {
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return fmt.Errorf("the pipelines are not listed: %w", err)
}
for _, key := range pkeys {
p, _, err := h.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || stepOfAsk(p, ask) < 0 {
continue
}
return h.endNotTaken(key, ask, bound)
}
return nil
}
func (h *Hearer) endNotTaken(key, ask string, bound time.Duration) error {
for {
p, rev, err := h.Store.Get(key)
if err != nil {
return fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
i := stepOfAsk(p, ask)
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning {
return nil
}
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Ended = seat.StepResultOutcomeCouldNotRun, &now
s.Why = fmt.Sprintf("the seat %s has no holder that took ask %s within %s", s.Seat, ask, bound)
p.Unlogged = append(p.Unlogged, endEntry(key, *s, now))
ended := endPipeline(&p, now)
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return fmt.Errorf("pipeline %s cannot be written: %w", key, err)
}
h.Failures.StepEnded(h.Store, p, *s)
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
}
h.appendUnlogged(key)
h.Ager.Age()
h.Fills.Check()
return h.said(p, false, ended)
}
}
// IsAskKey says whether a key of the state `bounds` is an ask's bound.
func IsAskKey(key string) bool { return strings.HasPrefix(key, "step-") && !strings.Contains(key, ".") }
// DropBoundPrefix begins the key of a superseded step's drop in the state `bounds`, `drop.<build seat>.<ask>` (design 47 G
// A-222, A-77): its five-minute expiry says the drop unanswered.
const DropBoundPrefix = "drop."
@@ -75,7 +75,7 @@ func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) {
}
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
said = nil
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil, nil); n != 1 || err != nil {
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil); n != 1 || err != nil {
t.Fatalf("supersede %d (%v)", n, err)
}
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
@@ -77,7 +77,18 @@ func startOf(p seat.Pipeline, now time.Time) time.Time {
// could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats'
// work queues, one per running step.
func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) {
return place(p, reg, entries, now, asker, true)
}
// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k).
// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in
// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and
// naming the run's ask and the step that asked it (A-469m, A-469q).
func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) {
entry := entries[p.Module]
candidate, named := CandidateOf(p)
share = share && named
askOfRun := map[string]int{} // a test-runs key to the index of its ask in asks
arts := append([]catalog.Artifact(nil), entry.Artifacts...)
sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref })
var steps []seat.StepResult
@@ -118,8 +129,27 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
}
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step)
body := stepAsk(p, entry, a, step, s.Ask, asker, now)
ask := Asked{Seat: bs, ID: s.Ask, Ask: body}
if share && BoundSeat(bs) {
digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch)
run := TestRunKey(candidate, step, digest)
reader := seat.TestRunReader{Pipeline: p.Pipeline, Artifact: a.Ref}
if j, asked := askOfRun[run]; asked {
// Another artifact's step of this pipeline asks the run: this one reads it.
first := &asks[j]
first.Readers = append(first.Readers, reader)
s.Ask, s.Outcome, s.On = "", seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}}
steps = append(steps, s)
continue
}
// A test run's steps wait for a build node from its ask until the holder's ask-taken (A-469b).
s.Shared, s.Outcome, s.On = []seat.Shared{{Run: run, Ask: s.Ask}}, seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader}
askOfRun[run] = len(asks)
}
steps = append(steps, s)
ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)}
if a.Package != nil {
ask.Package = a.Package.Name
}
@@ -156,6 +186,10 @@ type Asked struct {
// Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results
// are the ask's (§25, §416).
Package string
// Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline
// reading it, the asking step first (§724, §725); empty for an ask of no test run.
Run, Level, Digest string
Readers []seat.TestRunReader
}
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
@@ -170,8 +204,9 @@ func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step,
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
}
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put.
func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) error {
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
// holder with no bound of the runner's own (§535, §647 as A-466y amends them).
func askAll(asker StepAsker, key string, asks []Asked, recovery bool) error {
var first error
for _, a := range asks {
accept, ok := acceptStep[a.Seat]
@@ -179,15 +214,6 @@ func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) e
accept = to
}
err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
if ok {
// A build seat's holder says ask-taken: the ask's bound is armed before it is put (§535, §647).
if arm != nil && BoundSeat(a.Seat) {
if aerr := arm(a.ID); aerr != nil {
err = fmt.Errorf("its bound is not armed: %w", aerr)
ok = false
}
}
}
if ok {
err = asker.AskStep(accept, a.ID, a.Ask)
}
@@ -328,3 +328,207 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks))
}
}
// testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read.
type testRunsMem map[string]struct {
r Run
rev uint64
}
func (m testRunsMem) Create(key string, r Run) error {
if _, ok := m[key]; ok {
return ErrExists
}
m[key] = struct {
r Run
rev uint64
}{r, 1}
return nil
}
func (m testRunsMem) Get(key string) (Run, uint64, error) {
e, ok := m[key]
if !ok {
return Run{}, 0, ErrNoRun
}
e.r.Readers = append([]RunReader(nil), e.r.Readers...)
return e.r, e.rev, nil
}
func (m testRunsMem) Delete(key string, rev uint64) error {
if e, ok := m[key]; ok && e.rev != rev {
return ErrChanged
}
delete(m, key)
return nil
}
func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) {
if m[key].rev != rev {
return 0, ErrChanged
}
m[key] = struct {
r Run
rev uint64
}{r, rev + 1}
return rev + 1, nil
}
// One test run per candidate, level and tested packages (design 47 G §724, A-469c): a module whose program and tools bundle
// test one directory asks one lint, one unit and one integration step, the other artifact's steps reading them; a second
// pipeline of the same candidate reads the runs; a run ended could not run is asked again; a second revision with the same
// digests asks its own runs.
func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
reg := catalog.Registration{Module: "app", Branches: []string{"main"}}
entries := map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
}
store, runs, asks := memStore{}, testRunsMem{}, &stepAsks{}
s := &Starter{Pipelines: memCreated{store}, Store: store, TestRuns: runs, Say: func(string, any) error { return nil }, Steps: asks,
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
start := func(source, revision int64) seat.Pipeline {
t.Helper()
if ok, err := s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
t.Fatalf("start %d = %v, %v", source, ok, err)
}
p, _, _ := store.Get(Key("main", source, "app"))
return p
}
testAsks := func(from int) []string {
var out []string
for _, a := range asks.asks[from:] {
if b, ok := a.(nodebuildgo.StepAsk); ok {
out = append(out, b.Step.Artifact+" "+b.Step.Step)
}
}
return out
}
p := start(4, 9)
if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" {
t.Fatalf("the test steps asked = %q, want the program's three alone", got)
}
for _, level := range buildSteps {
asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level)
if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil {
t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker)
}
if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeWaitingForABuildNode || reader.On != seat.WaitOnBuildNode || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{
Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil ||
*reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader)
}
r, _, err := runs.Get(asker.Shared[0].Run)
if err != nil || r.Ask != asker.Ask || r.Level != level || len(r.Readers) != 2 || !strings.HasPrefix(asker.Shared[0].Run, "main.9."+level+".") {
t.Errorf("the run of %s = %+v, %v: want the program's ask and both artifacts reading it", level, r, err)
}
}
// A second pipeline of the same candidate (another source revision of the same catalog revision) reads the runs.
n := len(asks.asks)
q := start(5, 9)
if got := testAsks(n); len(got) != 0 {
t.Errorf("the second pipeline of the candidate asked %q, want it to read the runs", got)
}
unit := stepNamed(q, "app:program", "unit")
if unit.Ask != "" || unit.Shared[0].Ask != stepNamed(p, "app:program", "unit").Ask || *unit.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
t.Errorf("the second pipeline's unit step = %+v, want it reading the first's run", unit)
}
if r, _, _ := runs.Get(unit.Shared[0].Run); len(r.Readers) != 4 {
t.Errorf("the unit run's readers = %+v, want both pipelines' two artifacts", r.Readers)
}
// A run ended could not run is never read: the next step of its key asks it again.
key := stepNamed(p, "app:program", "lint").Shared[0].Run
r, rev, _ := runs.Get(key)
ended := time.Unix(9, 0)
r.Ended, r.Outcome = &ended, seat.StepResultOutcomeCouldNotRun
_, _ = runs.Update(key, r, rev)
n = len(asks.asks)
start(6, 9)
if got := strings.Join(testAsks(n), "|"); got != "app:program lint" {
t.Errorf("after a lint run that could not run, asked %q, want the lint step alone", got)
}
// A second revision with the same digests asks its own runs, so a failed test runs again.
n = len(asks.asks)
start(7, 10)
if got := len(testAsks(n)); got != 3 {
t.Errorf("a second revision asked %d test steps, want its own three", got)
}
}
// A seat with no holder is read only from the seat-holders view (design 47 G §535, §647 as A-466y and A-466z amend them):
// once that view's first view is read, a build step whose seat it names no holder for ends could not run naming the seat,
// and the pipeline ends failed at once naming it, every other step `not asked`, each said, none asked (A-470g, A-470h);
// before it is read, or with a holder named, the step is asked and waits with no bound of the runner's own; the claimant of
// the view is the one module the catalog halves name.
func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) {
reg, entries := placeFixture()
for _, c := range []struct {
name string
holders []string
read bool
asked bool
}{{"the view not read yet", nil, false, true}, {"a holder named", []string{"mesh-build-go"}, true, true}, {"no holder named", nil, true, false}} {
t.Run(c.name, func(t *testing.T) {
store, asks := memStore{}, &stepAsks{}
var said []int
s := &Starter{Pipelines: memCreated{store}, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
Registration: func(string) (catalog.Registration, bool) { return reg, true },
Holders: func(string) ([]string, bool) { return c.holders, c.read },
Ended: func(_ seat.Pipeline, steps []int) error { said = steps; return nil }}
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
p, _, _ := store.Get("main.4.app")
lint := stepNamed(p, "app:program", "lint")
asked := false
for _, id := range asks.ids {
asked = asked || id == lint.Ask
}
if c.asked && (lint.Outcome != seat.StepResultOutcomeRunning || !asked) {
t.Errorf("the lint step = %+v, asked %v; want it running and asked", lint, asked)
}
if c.asked {
if p.Outcome != seat.PipelineOutcomeRunning || said != nil {
t.Errorf("the pipeline = %s (%s), ended steps said %v; want it running", p.Outcome, p.Why, said)
}
return
}
if lint.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(lint.Why, "node-build-go has no holder") || len(asks.ids) != 0 {
t.Errorf("the lint step = %+v, %d asked; want could not run naming the seat, nothing asked", lint, len(asks.ids))
}
if want := "the seat node-build-go has no holder: the seat-holders view names none"; p.Outcome != seat.PipelineOutcomeFailed || p.Why != want || p.Ended == nil || p.EndCount != 1 {
t.Errorf("the pipeline = %s (%q), ended %v, count %d; want failed at once, %q", p.Outcome, p.Why, p.Ended, p.EndCount, want)
}
ended := 0
for _, st := range p.Steps {
switch {
case st.Seat == "node-build-go", st.Outcome == seat.StepResultOutcomeNotAsked:
ended++
case st.Outcome == seat.StepResultOutcomeBypassed, st.Outcome == seat.StepResultOutcomeCouldNotRun && st.Seat == "":
// ended by Place itself: a bypass, or a language no build seat names
default:
t.Errorf("the step %s %s = %s; want not asked", st.Artifact, st.Step, st.Outcome)
}
}
if ended < 3 || len(said) != ended {
t.Errorf("%d steps ended at placing, %d said; want each said", ended, len(said))
}
})
}
claims := func(seats ...string) map[string]any {
var cs []any
for _, n := range seats {
cs = append(cs, map[string]any{"name": n, "scope": "mesh"})
}
return map[string]any{"module": "x", "version": "0.1", "claims": cs}
}
half := Half{Entries: map[string]catalog.CatalogEntry{"assigner": {Module: "assigner", Manifest: claims("mesh-module-assigner", "mesh-wants")},
"app": {Module: "app", Manifest: claims()}}}
if got, ok := ClaimantOf([]Half{half, {}}, "mesh-module-assigner"); !ok || got != "assigner" {
t.Errorf("the claimant = %q, %v", got, ok)
}
other := Half{Entries: map[string]catalog.CatalogEntry{"other": {Module: "other", Manifest: claims("mesh-module-assigner")}}}
if got, ok := ClaimantOf([]Half{half, other}, "mesh-module-assigner"); ok {
t.Errorf("halves that disagree name %q", got)
}
}
@@ -95,12 +95,12 @@ func (r Reads) Stands() (seat.StandsAnswer, error) {
if s.Ended != nil || s.Started == nil {
continue
}
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
if !open(s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit,
Group: p.Group, Run: p.Run, Source: p.Source, Revision: p.Revision, Fingerprint: p.Fingerprint, Step: s.Step,
Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1})
Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1, Ask: askOfWait(s), Shared: sharedOfWait(s)})
}
}
sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) })
@@ -219,3 +219,19 @@ func startedOf(p seat.Pipeline) time.Time {
}
return *p.Started
}
// askOfWait is a waiting step's own ask (design 47 G §731, §496): a step reading another's test run keeps none (A-469m).
func askOfWait(s seat.StepResult) string { return s.Ask }
// sharedOfWait is the run a waiting step waits on, where the runner knows it (§731; A-469r, A-469v): a test step's run key
// and ask, as one entry with no machine and no asker.
func sharedOfWait(s seat.StepResult) []seat.Shared {
if len(s.Shared) == 0 {
return nil
}
out := make([]seat.Shared, 0, len(s.Shared))
for _, x := range s.Shared {
out = append(out, seat.Shared{Run: x.Run, Ask: x.Ask, Machine: x.Machine})
}
return out
}
@@ -7,6 +7,7 @@ import (
"testing"
"time"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
@@ -150,3 +151,55 @@ func TestShowAndKeptCarryTheStoresFill(t *testing.T) {
t.Errorf("show with the fill unread: %v", err)
}
}
// A step that reads a test run says its own step-started at every place it is asked, naming the run's key and ask in shared
// and no asker (design 47 G §726, §729; A-469a, A-469q), and stands names each waiting step's own ask and, for a test step,
// the run it waits on (§731; A-469r, A-469v).
func TestStandsAndStepStartedNameWhatAReadingStepWaitsOn(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
asker, reader := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:tools", "unit")
started := map[string]seat.StepResult{}
for _, e := range f.said {
started[e.Result.Artifact+" "+e.Result.Step] = e.Result
}
for _, s := range []seat.StepResult{asker, reader} {
got, ok := started[s.Artifact+" unit"]
if !ok || len(got.Shared) != 1 || got.Shared[0] != (seat.Shared{Run: asker.Shared[0].Run, Ask: asker.Ask}) {
t.Errorf("the step-started of %s = %+v (said %v), want the run's key and ask, no asker", s.Artifact, got, ok)
}
}
if _, ok := started[" composition"]; !ok {
t.Error("the composition step's step-started is not said")
}
pipes := memKept{}
put(t, pipes, p.Pipeline, f.pipeline(p))
a, err := Reads{Pipelines: pipes, Kept: memKept{}}.Stands()
if err != nil {
t.Fatal(err)
}
waits := map[string]seat.Wait{}
for _, w := range a.Stands {
waits[w.Artifact+" "+w.Step] = w
}
want := []seat.Shared{{Run: asker.Shared[0].Run, Ask: asker.Ask}}
if w := waits["app:program unit"]; w.Ask != asker.Ask || len(w.Shared) != 1 || w.Shared[0] != want[0] {
t.Errorf("the asking step stands as %+v, want its ask and its run", w)
}
if w := waits["app:tools unit"]; w.Ask != "" || len(w.Shared) != 1 || w.Shared[0] != want[0] {
t.Errorf("the reading step stands as %+v, want no ask of its own and the run's key and ask", w)
}
if w := waits[" composition"]; w.Ask == "" || len(w.Shared) != 0 {
t.Errorf("the composition step stands as %+v, want its own ask and no run", w)
}
// Resume asks the steps again and says their step-started again (A-469a).
n := len(f.said)
if _, err := f.s.Resume(f.store, "main", Half{Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}); err != nil {
t.Fatal(err)
}
if len(f.said) == n {
t.Error("resume said no step-started")
}
}
@@ -2,12 +2,14 @@ package pipeline
import (
"context"
"encoding/json"
"fmt"
"sort"
"sync"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Status is a step's status for one artifact of a module, read from the module's registration and never from a
@@ -212,3 +214,35 @@ func (r *Readiness) Observe(rev int64) (first, advanced bool) {
}
return false, false
}
// ClaimantOf is the module the catalog halves name as the claimant of seatName (design 47 G §614, A-82): the one module whose
// entry's manifest claims it, the same in every half that names one; none, more than one, or halves that disagree name none.
func ClaimantOf(halves []Half, seatName string) (string, bool) {
found := ""
for _, h := range halves {
var named []string
for _, e := range h.Entries {
raw, err := json.Marshal(e.Manifest)
if err != nil {
continue
}
m, err := catalogue.Decode(raw)
if err != nil {
continue
}
for _, c := range m.Claims {
if c.Name == seatName {
named = append(named, e.Module)
}
}
}
switch {
case len(named) == 0:
continue
case len(named) > 1, found != "" && found != named[0]:
return "", false
}
found = named[0]
}
return found, found != ""
}
@@ -229,9 +229,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
}
p.Fingerprint, p.Repository, p.Revision = e.Fp, e.Repository, revision
var asks []Asked
var ended []int
if s.Registration != nil {
if reg, ok := s.Registration(p.Module); ok {
p.Steps, asks = Place(p, reg, half.Entries, now, s.Name)
asks, ended = s.holderless(&p, asks, now)
} else {
s.logf("%s has no registration yet: its pipeline is placed with no step", key)
}
@@ -246,9 +248,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, p.Commit), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
if len(ended) > 0 {
return true, s.endedAtPlacing(p, ended)
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
if err := s.askSteps(key, p.Commit, asks); err != nil {
return true, err
}
}
@@ -79,7 +79,13 @@ func TestRunOnRequest(t *testing.T) {
if _, _, err := store.Get(RunKey("fix/1", a.Run, "same")); err == nil {
t.Error("an entry the members do not change got a pipeline")
}
if len(said) != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 {
// Each pipeline placed says pipeline-started, then step-started for each step it asks (A-469a).
started, steps := 0, 0
for _, x := range said {
started += strings.Count(x, seat.EventPipelineStarted+".")
steps += strings.Count(x, seat.EventStepStarted+".")
}
if started != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 || steps < len(asks.ids) {
t.Errorf("placing said %v and asked %d steps", said, len(asks.ids))
}
if ids, _ := s.UnplacedRuns(); len(ids) != 0 {
@@ -0,0 +1,488 @@
package pipeline
import (
"errors"
"fmt"
"strings"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// RunSettings are the runner's settings of its test runs (design 47 G §725; A-469d, A-469w, A-469x): the cap of test-runs
// the process judges a write by, how long an ended run is kept, the bound of a taken run's owed progress and the bound of
// an ask never taken.
type RunSettings struct {
CapBytes int64
Kept time.Duration
StepBound time.Duration
Untaken time.Duration
}
// The settings' defaults (§725; A-466n, A-466s): 16 MiB, 7 days, 30 minutes, 24 hours.
var DefaultRunSettings = RunSettings{CapBytes: 16 << 20, Kept: 7 * 24 * time.Hour, StepBound: 30 * time.Minute, Untaken: 24 * time.Hour}
// The two bound keys of a test run in test-runs, each with its own time to live (§725; A-468b, A-470): `bound.<run key>`,
// written at the build seat's ask-taken of the run's ask and again at each step-progress of it, and `untaken.<run key>`,
// written at the run's start.
const (
BoundPrefix = "bound."
UntakenPrefix = "untaken."
)
// RunObjects is the state test-runs' own object store, test-runs-objects (A-469e): a run's output, kept while its key lives.
type RunObjects interface {
Put(name string, data []byte) error
// Get reads an object; ErrNoRun when it is gone.
Get(name string) ([]byte, error)
Delete(name string) error
}
// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live, each holding the ask of the
// run it bounds (A-470n).
type RunClock interface {
// Bound writes key again holding ask, with ttl as its time to live, by compare-and-set on the key's latest revision
// (retried when another write came between); it answers the revision written.
Bound(key, ask string, ttl time.Duration) (uint64, error)
// Bounded reads key: the ask it holds and its revision; ok false when it is not written.
Bounded(key string) (ask string, revision uint64, ok bool, err error)
// Unbound deletes key; a key gone is no fault.
Unbound(key string) error
// UnboundAt deletes key at revision; a later write of it (another run's bound) is kept, and a key gone is no fault.
UnboundAt(key string, revision uint64) error
// Has says whether key is written.
Has(key string) (bool, error)
// Keys are test-runs' keys.
Keys() ([]string, error)
}
// ParseRunBound reads a key of test-runs as a run's bound: the run's key and whether it is the untaken bound.
func ParseRunBound(key string) (run string, untaken, ok bool) {
if run, ok = strings.CutPrefix(key, BoundPrefix); ok {
return run, false, run != ""
}
if run, ok = strings.CutPrefix(key, UntakenPrefix); ok {
return run, true, run != ""
}
return "", false, false
}
// IsRunBound says whether a key of test-runs is a bound key rather than a run.
func IsRunBound(key string) bool {
_, _, ok := ParseRunBound(key)
return ok
}
// runOfAsk is the test run asked under ask: the run its asking step names in Shared, read from the state pipelines.
func (h *Hearer) runOfAsk(ask string) (string, bool) {
if h.TestRuns == nil || ask == "" {
return "", false
}
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return "", false
}
for _, key := range pkeys {
p, _, err := h.Store.Get(key)
if err != nil {
continue
}
if i := stepOfAsk(p, ask); i >= 0 {
run := sharedRun(p.Steps[i])
return run, run != ""
}
}
return "", false
}
// RunTaken takes the ask-taken of a test run's ask (§725, A-470): the node that took it is kept on the run, its untaken
// bound deleted and its clock begun, `bound.<run key>` written with test-step-bound as its time to live. An ask of no test
// run takes nothing.
func (h *Hearer) RunTaken(ask, node string) error {
run, ok := h.runOfAsk(ask)
if !ok {
return nil
}
for {
r, rev, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) {
return nil
}
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
if r.Ask != ask || r.Ended != nil {
h.logf("the ask-taken of ask %s is late: the test run %s names ask %s (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
return nil
}
if r.Node != node {
r.Node = node
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return fmt.Errorf("the test run %s is not written: %w", run, err)
}
}
h.runStarted(run, r)
break
}
if err := armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound); err != nil {
return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err)
}
// The untaken bound is deleted only while it holds this ask: a rerun's own untaken bound is kept.
held, rev, ok, err := h.Clock.Bounded(UntakenPrefix + run)
if err != nil || !ok || held != ask {
return err
}
return h.Clock.UnboundAt(UntakenPrefix+run, rev)
}
// armBound writes the bound key of the run under ask holding that ask, judged from a read of the run, then reads the run
// again (A-470n): when it no longer names ask, or has ended, the bound just written is deleted at its revision, so no
// bound written from the earlier read ends the run after it, and a bound another run wrote since is kept.
func armBound(clock RunClock, runs TestRuns, key, run, ask string, ttl time.Duration) error {
rev, err := clock.Bound(key, ask, ttl)
if err != nil {
return err
}
r, _, err := runs.Get(run)
if err != nil && !errors.Is(err, ErrNoRun) {
return err
}
if errors.Is(err, ErrNoRun) || r.Ended != nil || r.Ask != ask {
return clock.UnboundAt(key, rev)
}
return nil
}
// RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that
// keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing.
func (h *Hearer) RunProgress(ask string) error {
run, ok := h.runOfAsk(ask)
if !ok {
return nil
}
r, _, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) {
h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
return nil
}
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
return armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound)
}
// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not
// run`, naming its ask and the bound, its ask dropped and every reader ended with it. An untaken bound of a run whose ask
// was taken, and a bound of a run ended already, end nothing.
func (h *Hearer) RunExpired(key string) error {
run, untaken, ok := ParseRunBound(key)
if !ok || h.TestRuns == nil {
return nil
}
r, _, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) {
return nil
}
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
// An untaken bound of a taken run, and a clock of a run not taken (a clock left from an ask before a rerun's), end nothing.
if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") {
return nil
}
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again,
// so an expiry heard after the bound was armed again for the run's own ask ends nothing.
if _, _, armed, err := h.Clock.Bounded(key); err != nil {
return fmt.Errorf("the bound %s is not read again: %w", key, err)
} else if armed {
h.logf("the expiry of %s ends nothing: the bound is armed again (A-470o)", key)
return nil
}
s := h.runSettings()
why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound)
if untaken {
why = fmt.Sprintf("the ask %s of %s was not taken within %s of the run's start (run-untaken-bound)", r.Ask, r.Seat, s.Untaken)
}
if h.DropRun != nil {
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err)
}
}
return h.EndRun(run, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil)
}
// TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an
// untaken run whose untaken bound is gone, passed its bound while the runner was away and ends as its expiry would. It
// says how many it ended.
func (h *Hearer) TakeUp() (int, error) {
if h.TestRuns == nil || h.Clock == nil {
return 0, nil
}
all, err := h.Clock.Keys()
if err != nil {
return 0, fmt.Errorf("the test runs are not listed: %w", err)
}
ended := 0
var first error
for _, run := range all {
if IsRunBound(run) {
continue
}
r, _, err := h.TestRuns.Get(run)
if err != nil || r.Ended != nil {
continue
}
bound := UntakenPrefix + run
if r.Node != "" {
bound = BoundPrefix + run
}
has, err := h.Clock.Has(bound)
if err != nil || has {
if err != nil && first == nil {
first = err
}
continue
}
if err := h.RunExpired(bound); err != nil && first == nil {
first = err
}
ended++
}
return ended, first
}
func (h *Hearer) runSettings() RunSettings {
s := h.Runs
d := DefaultRunSettings
if s.CapBytes <= 0 {
s.CapBytes = d.CapBytes
}
if s.Kept <= 0 {
s.Kept = d.Kept
}
if s.StepBound <= 0 {
s.StepBound = d.StepBound
}
if s.Untaken <= 0 {
s.Untaken = d.Untaken
}
return s
}
// EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set,
// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not
// ended again, and end's ask, the one its caller judged, ends only a run under that ask: a run a rerun asked since is not
// ended by a judgement of the one before.
func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error {
var r Run
for {
got, rev, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) {
return nil
}
if err != nil {
return fmt.Errorf("the test run %s is not read: %w", run, err)
}
if got.Ended != nil || (end.Ask != "" && got.Ask != end.Ask) {
return nil
}
now := h.now()
got.Ended, got.Outcome, got.Said, got.Tests = &now, end.Outcome, end.Said, end.Tests
if end.Node != "" {
got.Node = end.Node
}
if end.Output != "" {
got.Output = end.Output
}
if _, err := h.TestRuns.Update(run, got, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return fmt.Errorf("the test run %s is not written: %w", run, err)
}
r = got
break
}
if h.Clock != nil {
for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} {
if err := h.Clock.Unbound(k); err != nil {
h.logf("the bound %s of the ended test run is not deleted: %v", k, err)
}
}
}
var first error
for _, reader := range r.Readers {
if skip != nil && reader.Of() == *skip {
continue
}
if err := h.endReader(reader, run, r); err != nil && first == nil {
first = err
}
}
return first
}
// endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming
// the run in Shared as it did. A step ended already takes nothing, but one the rerun that asked the run marked (A-470d,
// A-470e): it read another ask of the key, and it moves with this one, said again marked a rerun by that caller, naming
// the new ask, its pipeline reopened (§727).
func (h *Hearer) endReader(reader RunReader, run string, r Run) error {
for {
p, rev, err := h.Store.Get(reader.Pipeline)
if err != nil {
return nil // a pipeline gone has no reader left to end
}
i, moved := -1, false
for j, s := range p.Steps {
if s.Artifact != reader.Artifact || sharedRun(s) != run {
continue
}
if open(s) {
i, moved = j, false
} else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil {
i, moved = j, true
}
}
if i < 0 {
return nil
}
now := h.now()
s := &p.Steps[i]
if moved {
s.Rerun, s.By = true, reader.By
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: s.Shared[0].AskedBy}}
if p.Outcome != seat.PipelineOutcomeRunning {
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
}
}
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = r.Outcome, r.Said, r.Node, &now, nil
if r.Tests != nil {
t := *r.Tests
s.Tests = &t
}
p.Unlogged = append(p.Unlogged, endEntry(reader.Pipeline, *s, now))
asked, err := askApproval(reader.Pipeline, &p, now, h.Asker)
if err != nil {
return err
}
ended := endPipeline(&p, now)
if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err)
}
h.Failures.StepEnded(h.Store, p, *s)
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err)
}
h.appendUnlogged(reader.Pipeline)
return h.said(p, asked, ended)
}
}
// NoReaderLeft is what a test run whose last reader left ends with (A-469z): its ask is dropped, and a later step of the
// candidate starts a new run.
const NoReaderLeft = "no reader is left"
// leaveRun takes the step of pipeline key that read or asked a test run out of the run's readers (§726): a step superseded
// or cancelled. When no reader is left the run ends `could not run`, naming that (A-469z), and its ask is dropped (§119);
// while one is left the ask stays, so a superseded pipeline never takes away another's run. A step of no test run leaves
// nothing.
func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) {
run := sharedRun(s)
if run == "" || h.TestRuns == nil {
return
}
reader := seat.TestRunReader{Pipeline: key, Artifact: s.Artifact}
for {
r, rev, err := h.TestRuns.Get(run)
if errors.Is(err, ErrNoRun) {
return
}
if err != nil {
h.logf("the test run %s is not read, so the step %s of %s does not leave it: %v", run, s.Step, key, err)
return
}
left := r.Readers[:0:0]
for _, o := range r.Readers {
if o.Of() != reader {
left = append(left, o)
}
}
if len(left) == len(r.Readers) {
return
}
r.Readers = left
// The last to leave ends the run in the same write, so a step joining it after that read finds it ended and asks
// its own, never joining a run then ended for having no reader.
last := len(left) == 0 && r.Ended == nil
if last {
now := h.now()
r.Ended, r.Outcome, r.Said = &now, seat.StepResultOutcomeCouldNotRun, NoReaderLeft
}
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err)
return
}
if !last {
return
}
if h.Clock != nil {
for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} {
if err := h.Clock.Unbound(k); err != nil {
h.logf("the bound %s of the ended test run is not deleted: %v", k, err)
}
}
}
if h.DropRun != nil {
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err)
}
}
return
}
}
// open says whether a step is asked and not ended: running, or a test step waiting for a build node to take its run's ask
// (A-469b).
func open(s seat.StepResult) bool {
return s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode
}
// runStarted moves each step reading run from waiting for a build node to running, the holder's ask-taken heard (A-469b),
// and says its step-started again, so the watchman's build-capacity waits let it go (§86, §732).
func (h *Hearer) runStarted(run string, r Run) {
for _, reader := range r.Readers {
for {
p, rev, err := h.Store.Get(reader.Pipeline)
if err != nil {
break
}
var moved []int
for j, s := range p.Steps {
if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeWaitingForABuildNode {
p.Steps[j].Outcome, p.Steps[j].On = seat.StepResultOutcomeRunning, ""
moved = append(moved, j)
}
}
if len(moved) == 0 {
break
}
if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
h.logf("the steps of %s reading the taken test run %s are not written running: %v", reader.Pipeline, run, err)
break
}
for _, j := range moved {
if err := h.say(seat.EventStepStarted, p, &p.Steps[j]); err != nil {
h.logf("step-started of %s's %s is not said: %v", reader.Pipeline, p.Steps[j].Step, err)
}
}
break
}
}
}
@@ -4,8 +4,10 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
@@ -73,8 +75,12 @@ type Starter struct {
// Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when
// none was heard, which asks the step with none.
Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error)
// Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none.
Arm Arm
// Holders are the modules the seat mesh-module-assigner's view seat-holders names for a seat, and whether that view's
// first view is read (§535, §647 as A-466y, A-466z amend them); nil, or a view not read yet, judges no seat holderless.
Holders func(seat string) ([]string, bool)
// Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end
// (Hearer.EndedAtPlacing); nil says it in the journal alone.
Ended func(p seat.Pipeline, steps []int) error
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
Fills *Fills
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
@@ -83,6 +89,116 @@ type Starter struct {
Store Updater
// Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none.
Supersede func(newer seat.Pipeline)
// TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest
// (§724, §725); nil shares none, each step's ask put. Clock writes a run's untaken bound at its start, Runs holds the
// settings, RunsFill reads test-runs' fill, judged against Runs' cap before a run is written (A-469d), and EndReaders ends
// the steps of a run read ended or never asked.
TestRuns TestRuns
Clock RunClock
Runs RunSettings
RunsFill func() (seat.Fill, error)
EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error
}
// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view
// seat-holders names no holder for (design 47 G §32, §535, §647 as A-466y and A-466z amend them; A-470g, A-470h): judged only
// once that view's first view is read, never from the picture before it. Each such step ends `could not run` naming the seat,
// every other step not ended ends `not asked`, each with its step log entry, and the pipeline ends `failed` naming the seat;
// none of its steps is asked. It answers the asks still to put and the steps it ended, none when no seat is holderless.
func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]Asked, []int) {
if s.Holders == nil {
return asks, nil
}
var none []string
for _, st := range p.Steps {
if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) {
continue
}
if holders, read := s.Holders(st.Seat); read && len(holders) == 0 {
none = append(none, st.Seat)
}
}
if len(none) == 0 {
return asks, nil
}
sort.Strings(none)
whys := make([]string, 0, len(none))
for _, n := range none {
whys = append(whys, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", n))
}
var ended []int
for i := range p.Steps {
st := &p.Steps[i]
if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting {
continue
}
st.Ended, st.Ask, st.Shared = &now, "", nil
if slices.Contains(none, st.Seat) {
st.Outcome, st.Why = seat.StepResultOutcomeCouldNotRun, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", st.Seat)
} else {
st.Outcome, st.Why = seat.StepResultOutcomeNotAsked, "the pipeline failed at once: "+strings.Join(whys, "; ")
}
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
ended = append(ended, i)
}
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, strings.Join(whys, "; "), &now
p.EndCount++
return nil, ended
}
// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the
// pipeline's end, through Ended; with none given it is said in the journal alone.
func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error {
s.logf("%s ended failed at once: %s (A-470g)", p.Pipeline, p.Why)
if s.Ended == nil {
return nil
}
return s.Ended(p, steps)
}
// askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then
// the kept results and toolchain are carried and each ask is put.
func (s *Starter) askSteps(key, commit string, asks []Asked) error {
placed := asks
asks = s.share(key, asks)
var err error
if len(asks) > 0 {
s.carry(key, asks)
err = askAll(s.Steps, key, asks, s.recovered(key, commit))
}
s.sayStarted(key, placed)
return err
}
// sayStarted says step-started for each step of pipeline key that asks were placed for (design 47 G §726, §729; A-469a):
// the step asked, and each step reading the test run it asks, naming the run's key and ask in shared, so the watchman times
// it from that ask (§496, §732). A composition step's, said before any run is chosen, names none (§729).
func (s *Starter) sayStarted(key string, asks []Asked) {
if s.Store == nil || s.Say == nil || len(asks) == 0 {
return
}
ids, runs := map[string]bool{}, map[string]bool{}
for _, a := range asks {
ids[a.ID] = true
if a.Run != "" {
runs[a.Run] = true
}
}
p, _, err := s.Store.Get(key)
if err != nil {
s.logf("the pipeline %s is not read, so its steps' step-started is not said: %v", key, err)
return
}
at := s.now()
for _, st := range p.Steps {
if !open(st) || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) {
continue
}
subject, ev := stepEvent(seat.EventStepStarted, p, st, at)
if err := s.Say(subject, ev); err != nil {
s.logf("step-started of %s's %s %s is not said: %v", key, st.Artifact, st.Step, err)
}
}
}
func newPipeline(b Begin, key string, now time.Time) seat.Pipeline {
@@ -115,11 +231,13 @@ func (s *Starter) Start(b Begin) (bool, error) {
}
p := newPipeline(b, key, now)
var asks []Asked
var ended []int
if b.unheld {
s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch)
} else if s.Registration != nil {
if reg, ok := s.Registration(b.Module); ok {
p.Steps, asks = Place(p, reg, b.Entries, now, s.Name)
asks, ended = s.holderless(&p, asks, now)
} else {
s.logf("%s has no registration yet: its pipeline starts with no step and waits on the registration of %s, which places them (A-327)", key, b.Module)
}
@@ -143,9 +261,11 @@ func (s *Starter) Start(b Begin) (bool, error) {
if s.Supersede != nil {
s.Supersede(p)
}
if len(ended) > 0 {
return true, s.endedAtPlacing(p, ended)
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, b.Commit)); err != nil {
if err := s.askSteps(key, b.Commit, asks); err != nil {
return true, err
}
}
@@ -187,24 +307,32 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
if !ok {
continue
}
running := map[string]bool{}
// A step that names its test run is asked as the run's asker (share); one that names none was asked before the test
// runs and keeps its own ask, ending by the path it was asked under (§730, A-468k).
running, own := map[string]bool{}, map[string]bool{}
for _, st := range p.Steps {
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
running[st.Ask] = true
if open(st) && st.Ask != "" {
running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0
}
}
_, placed := Place(p, reg, half.Entries, now, s.Name)
_, shared := Place(p, reg, half.Entries, now, s.Name)
_, unshared := place(p, reg, half.Entries, now, s.Name, false)
var due []Asked
for _, a := range placed {
if running[a.ID] {
for _, a := range shared {
if running[a.ID] && !own[a.ID] {
due = append(due, a)
}
}
for _, a := range unshared {
if running[a.ID] && own[a.ID] {
a.Run = ""
due = append(due, a)
}
}
if len(due) == 0 {
continue
}
s.carry(key, due)
if err := askAll(s.Steps, s.Arm, key, due, s.recovered(key, p.Commit)); err != nil && first == nil {
if err := s.askSteps(key, p.Commit, due); err != nil && first == nil {
first = err
}
asked += len(due)
@@ -284,6 +412,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
}
var asks []Asked
p.Steps, asks = Place(p, r, half.Entries, now, s.Name)
asks, ended := s.holderless(&p, asks, now)
if _, err := store.Update(key, p, rev); err != nil {
s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err)
continue
@@ -299,9 +428,14 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
s.logf("pipeline-started of %s is not said: %v", key, err)
}
}
if len(ended) > 0 {
if err := s.endedAtPlacing(p, ended); err != nil {
note(err)
}
continue
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
if err := s.askSteps(key, p.Commit, asks); err != nil {
note(err)
}
}
@@ -26,6 +26,9 @@ type StepSeat struct {
Prefix string
// Decode reads the seat's step-ended body; nil is the build seats' one form.
Decode func([]byte) (StepEnded, error)
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
// a seat whose ends name none.
Shared func([]byte) []seat.Shared
}
// StepSeats are the three build seats, from their contracts.
@@ -36,7 +39,25 @@ var StepSeats = []StepSeat{
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
// The seat mesh-wants answers the composition step (§22, §24).
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
}
// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d,
// A-469t): the same form, under the same name.
func wantsShared(raw []byte) []seat.Shared {
var w wants.StepEnded
if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 {
return nil
}
out := make([]seat.Shared, 0, len(w.Shared))
for _, x := range w.Shared {
e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask}
if x.AskedBy != nil {
e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact}
}
out = append(out, e)
}
return out
}
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
@@ -118,6 +139,26 @@ type Hearer struct {
Fills *Fills
// Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none.
Widens func(seat.Pipeline) []seat.Widening
// TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a
// run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run.
TestRuns TestRuns
Clock RunClock
Runs RunSettings
DropRun Drop
// RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v).
RunOutputs RunObjects
}
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
// of a run that was never asked.
func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error {
var first error
for _, reader := range readers {
if err := h.endReader(RunReader{Pipeline: reader.Pipeline, Artifact: reader.Artifact}, run, Run{TestRun: r}); err != nil && first == nil {
first = err
}
}
return first
}
func (h *Hearer) now() time.Time {
@@ -132,6 +173,13 @@ func (h *Hearer) now() time.Time {
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
return h.OnStepEndedShared(bs, subject, ev, nil)
}
// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
// A-469m).
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
node := strings.TrimPrefix(subject, bs.Prefix)
if node == subject || node == "" || strings.Contains(node, ".") {
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
@@ -153,7 +201,15 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
case !open(p.Steps[i]):
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
return p, h.endRunOf(p, p.Steps[i], ev)
}
// The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725).
h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why)
}
return p, nil
case !endOutcome(ev.Outcome):
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
@@ -161,6 +217,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
if len(shared) > 0 {
s.Shared = shared
}
if ev.Tests != nil {
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
@@ -177,6 +236,8 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
}
unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...)
p.Unlogged = append(p.Unlogged, unlogged)
// The steps of this pipeline reading the run end in the same write, so the pipeline ends once (§726, §727).
readers := h.sameRunReaders(&p, i, ev, now)
from := now
if ev.Ended != nil {
from = ev.Ended.UTC()
@@ -191,6 +252,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
} else if err != nil {
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
}
// The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a
// finding's event the bus is slow to take) delays it past its bound (A-470o).
runErr := h.endRunOf(p, *s, ev)
if h.Keep != nil {
for _, k := range ev.Kept {
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
@@ -209,14 +273,94 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
}
for _, j := range readers {
if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
}
}
// The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606).
h.appendUnlogged(ev.Pipeline)
h.Ager.Age()
h.Fills.Check()
return p, h.said(p, asked, ended)
if err := h.said(p, asked, ended); err != nil {
return p, err
}
return p, runErr
}
}
// sameRunReaders ends, in p, every other step reading the test run step i asked with ev, its ask's end: a running reader,
// and one the rerun that asked the run marked, which moves (A-470d, A-470e), said again marked a rerun by that caller. It
// answers their indexes.
func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.Time) []int {
asker := p.Steps[i]
run := sharedRun(asker)
if run == "" || h.TestRuns == nil || asker.Ask != ev.ID {
return nil
}
marks := map[string]RunReader{}
if r, _, err := h.TestRuns.Get(run); err == nil {
for _, o := range r.Readers {
if o.Pipeline == p.Pipeline {
marks[o.Artifact] = o
}
}
}
var out []int
for j := range p.Steps {
s := &p.Steps[j]
if j == i || sharedRun(*s) != run {
continue
}
switch m := marks[s.Artifact]; {
case open(*s) && s.Ask == "":
case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil:
s.Rerun, s.By = true, m.By
s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}}
default:
continue
}
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = asker.Outcome, asker.Why, asker.Node, &now, nil
if asker.Tests != nil {
t := *asker.Tests
s.Tests = &t
}
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now))
out = append(out, j)
}
return out
}
// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in
// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it.
// A step that asked no run ends nothing more.
func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error {
run := sharedRun(s)
if run == "" || h.TestRuns == nil || s.Ask != ev.ID {
return nil
}
end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
if ev.Tests != nil {
end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
if h.RunOutputs != nil {
// The run's output object holds the packages it ran with their digests and the build seat's output (§725).
out := RunOutput{Output: ev.Output, Packages: []seat.TestRunPackage{}}
for _, k := range append(append([]nodebuildgo.KeptResult(nil), ev.Kept...), ev.Carried...) {
out.Packages = append(out.Packages, seat.TestRunPackage{Package: k.Package, Digest: k.Digest})
}
raw, err := json.Marshal(out)
if err == nil {
err = h.RunOutputs.Put(run, raw)
}
if err != nil {
return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err)
}
end.Output = run
}
return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact})
}
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
if asked {
@@ -300,8 +444,8 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
at := h.now()
switch event {
case seat.EventStepEnded, seat.EventStepStarted:
return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step})
_, ev := stepEvent(event, p, *step, at)
return h.Say(subject, ev)
case seat.EventPipelineEnded:
e := pipelineEnded(p)
if h.Widens != nil {
@@ -314,17 +458,46 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
}
}
// stepEvent is a step's step-started or step-ended of p, on the runner's subject for it: its result as it stands, with
// its shared when it reads a run (§729; A-468e, A-469q).
func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) {
last := p.Commit
if p.Group != "" {
last = p.Group
}
if event == seat.EventStepStarted {
// A step-started names the run and its ask, never the step that asked it (A-469q).
var shared []seat.Shared
for _, x := range step.Shared {
x.AskedBy = nil
shared = append(shared, x)
}
step.Shared = shared
}
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
return subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: step}
}
// The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests.
const (
budgetUnit = 500
budgetIntegration = 50
)
// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any.
// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many
// steps read it (§726, A-466i); nil when no step counted any.
func testCounts(steps []seat.StepResult) *seat.TestCounts {
var t seat.TestCounts
counted := false
runs := map[string]bool{}
for _, s := range steps {
if run := sharedRun(s); run != "" && s.Seat != wants.Seat {
if runs[run] {
continue
}
runs[run] = true
}
if s.Tests != nil {
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
}
@@ -380,7 +553,7 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
// pipeline's end (A-104).
continue
case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning:
case s.Outcome == seat.StepResultOutcomeWaiting, open(s):
return false
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
@@ -401,3 +574,22 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
}
return true
}
// EndedAtPlacing says a pipeline its starter ended at placing, a seat with no holder (A-470g, A-470h): the step-ended of each
// step of steps, its step log entries appended, the head's finding pipeline-failed (§33), then its pipeline-ended.
func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error {
var first error
for _, i := range steps {
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {
first = fmt.Errorf("pipeline %s's %s is written but not said: %w", p.Pipeline, seat.EventStepEnded, err)
}
}
h.appendUnlogged(p.Pipeline)
if len(steps) > 0 {
h.Failures.StepEnded(h.Store, p, p.Steps[steps[0]])
}
if err := h.said(p, false, true); err != nil && first == nil {
first = err
}
return first
}
@@ -302,49 +302,6 @@ func TestAnOptionalStepStillWaitingHoldsNoEnd(t *testing.T) {
}
}
// An ask no holder took within its bound ends its step `could not run` naming the seat, and with the rest ended the pipeline
// failed (design 47 G §32, §535, §647, A-104); a step that ended, or ask nobody placed, ends nothing.
func TestAnAskNoHolderTookEndsItsStepNamingTheSeat(t *testing.T) {
h, store, said, _, _, _ := fixture()
e := store[pkey]
e.p.Steps[1].Outcome = seat.StepResultOutcomePassed // the unit step ended; only lint is waiting for a holder
e.p.Steps[2].Outcome = seat.StepResultOutcomeNotAsked
store[pkey] = e
if err := h.NotTaken("nobody-asked", AskBound); err != nil || store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning {
t.Fatalf("an ask no step names ended something: %v", err)
}
if err := h.NotTaken("a1", AskBound); err != nil {
t.Fatal(err)
}
p := store[pkey].p
if p.Steps[0].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(p.Steps[0].Why, nodebuildgo.Seat) ||
p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(strings.Join(*said, " "), seat.EventPipelineEnded) {
t.Errorf("step %+v, pipeline %s (%s), said %v: want could not run naming %s and a failed pipeline", p.Steps[0], p.Outcome, p.Why, *said, nodebuildgo.Seat)
}
// Its end taken once: the bound's expiry heard again changes nothing.
n := len(*said)
if err := h.NotTaken("a1", AskBound); err != nil || len(*said) != n {
t.Errorf("a second expiry said %d more events: %v", len(*said)-n, err)
}
}
// The bound is armed before the ask is put, only for the build seats whose holder says ask-taken, and an ask whose bound is
// not armed is not put (§535, §647).
func TestTheBoundIsArmedBeforeAnAskIsPut(t *testing.T) {
var order []string
asker := askerFunc(func(accept, id string, _ any) error { order = append(order, "put "+id); return nil })
arm := Arm(func(ask string) error { order = append(order, "arm "+ask); return nil })
asks := []Asked{{Seat: nodebuildgo.Seat, ID: "step-1", Ask: 1}, {Seat: wants.Seat, ID: "step-2", Ask: 2}}
if err := askAll(asker, arm, "k", asks, false); err != nil || strings.Join(order, ",") != "arm step-1,put step-1,put step-2" {
t.Errorf("order %v, %v; want the build seat's bound armed first and mesh-wants' ask with none", order, err)
}
order = nil
failing := Arm(func(string) error { return errors.New("bucket gone") })
if err := askAll(asker, failing, "k", asks[:1], false); err == nil || len(order) != 0 {
t.Errorf("an ask whose bound cannot be armed was put or said nothing: %v %v", order, err)
}
}
type askerFunc func(accept, id string, body any) error
func (f askerFunc) AskStep(accept, id string, body any) error { return f(accept, id, body) }
@@ -359,7 +316,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
var to []string
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
if err := askAll(asker, nil, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want {
if err := askAll(asker, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want {
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
}
}
@@ -30,7 +30,7 @@ func (h *Hearer) Taken(ask, node string) error {
break
}
i := stepOfAsk(p, ask)
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning || p.Steps[i].Node == node {
if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node {
break
}
p.Steps[i].Node = node
@@ -49,10 +49,10 @@ func (h *Hearer) Taken(ask, node string) error {
// whose fingerprint newer's commit changed, and for a publish's pipeline each an older publish of the package started
// (design 47 G §31, §119; ADR 0666 §9; A-202): the pipeline and each step not ended
// `superseded`, naming the newer commit, and it says pipeline-ended. A module the newer commit left as it was finishes its
// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop; its bound deleted through
// unbound); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on
// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop); a step of a test run leaves
// it, its ask dropped only with its last reader (§726); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on
// request's pipeline neither supersedes nor is superseded. It says how many pipelines it ended.
func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask string) error) (int, error) {
func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop) (int, error) {
if newer.Run != "" || newer.Source <= 0 {
return 0, nil
}
@@ -67,7 +67,7 @@ func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask stri
if key == newer.Pipeline {
continue
}
done, err := h.supersedeOne(key, newer, why, drop, unbound)
done, err := h.supersedeOne(key, newer, why, drop)
if err != nil && first == nil {
first = err
}
@@ -99,7 +99,7 @@ func supersededWhy(newer seat.Pipeline) string {
return fmt.Sprintf("the newer commit %s of %s changed %s's fingerprint", newer.Commit, newer.Branch, newer.Module)
}
func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop, unbound func(string) error) (bool, error) {
func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop) (bool, error) {
for {
p, rev, err := h.Store.Get(key)
if err != nil || !superseded(p, newer) {
@@ -110,10 +110,11 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
var changed []int
for i := range p.Steps {
s := &p.Steps[i]
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if s.Ask != "" && BoundSeat(s.Seat) {
// A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun).
if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" {
drops = append(drops, *s)
}
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now)
@@ -136,17 +137,15 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
if len(changed) > 0 {
h.appendUnlogged(key)
}
for _, i := range changed {
h.leaveRun(key, p.Steps[i], why)
}
for _, s := range drops {
if drop != nil {
if err := drop(s.Seat, s.Node, s.Ask, why); err != nil {
h.logf("the drop of ask %s of the superseded %s is not answered: %v", s.Ask, key, err)
}
}
if unbound != nil {
if err := unbound(s.Ask); err != nil {
h.logf("the bound of ask %s of the superseded %s is not dropped: %v", s.Ask, key, err)
}
}
}
if err := h.said(p, false, true); err != nil && first == nil {
first = err
@@ -29,7 +29,6 @@ func TestSupersede(t *testing.T) {
}
type dropped struct{ seat, node, ask string }
var drops []dropped
var unbound []string
drop := func(s, node, ask, why string) error {
if !strings.Contains(why, "c2") {
t.Errorf("the drop of %s names no newer commit: %q", ask, why)
@@ -37,7 +36,6 @@ func TestSupersede(t *testing.T) {
drops = append(drops, dropped{s, node, ask})
return nil
}
unbind := func(ask string) error { unbound = append(unbound, ask); return nil }
newer := func(key string, source int64, fp string, run string, pub *seat.Published) seat.Pipeline {
return seat.Pipeline{Pipeline: key, Branch: "main", Source: source, Module: "app", Commit: "c2", Fingerprint: fp, Run: run, Published: pub,
Outcome: seat.PipelineOutcomeRunning}
@@ -50,12 +48,12 @@ func TestSupersede(t *testing.T) {
{"a run on request", newer("main.r1.app", 6, "src2:bb", "r1", nil)},
{"an older source revision", newer("main.4.app", 4, "src2:bb", "", nil)},
} {
if n, err := h.Supersede(c.newer, drop, unbind); n != 0 || err != nil {
if n, err := h.Supersede(c.newer, drop); n != 0 || err != nil {
t.Errorf("%s superseded %d (%v)", c.name, n, err)
}
}
*said = nil
if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop, unbind); n != 1 || err != nil {
if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop); n != 1 || err != nil {
t.Fatalf("a changed fingerprint superseded %d (%v)", n, err)
}
p, _, _ := store.Get(pkey)
@@ -68,13 +66,13 @@ func TestSupersede(t *testing.T) {
}
}
want := []dropped{{nodebuildgo.Seat, "n7", "a1"}, {nodebuildgo.Seat, "", "a2"}}
if !slices.Equal(drops, want) || !slices.Equal(unbound, []string{"a1", "a2"}) {
t.Errorf("dropped %+v and unbound %v, want %+v (the mesh-builds step only ended)", drops, unbound, want)
if !slices.Equal(drops, want) {
t.Errorf("dropped %+v, want %+v (the mesh-builds step only ended)", drops, want)
}
if n := len(*said); n != 5 || (*said)[4] != seat.EventPipelineEnded+".app.main.c1" {
t.Errorf("said %v, want four step-ended then pipeline-ended", *said)
}
if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop, unbind); n != 0 {
if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop); n != 0 {
t.Errorf("a superseded pipeline was superseded again: %d", n)
}
@@ -82,10 +80,10 @@ func TestSupersede(t *testing.T) {
pub := memStore{"main.5.lib": {p: seat.Pipeline{Pipeline: "main.5.lib", Branch: "main", Source: 5, Module: "lib", Fingerprint: "src2:aa",
Outcome: seat.PipelineOutcomeRunning, Published: &seat.Published{Artifact: "core:pkg", Version: "v1"}}, rev: 1}}
h.Store = pub
if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop, unbind); n != 0 {
if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop); n != 0 {
t.Errorf("a publish of the same version superseded %d", n)
}
if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop, unbind); n != 1 || err != nil {
if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop); n != 1 || err != nil {
t.Fatalf("a newer publish superseded %d (%v)", n, err)
}
if p, _, _ := pub.Get("main.5.lib"); p.Outcome != seat.PipelineOutcomeSuperseded || !strings.Contains(p.Why, "v2") {
@@ -105,9 +103,6 @@ func TestBoundDrop(t *testing.T) {
if _, _, ok := ParseDropBound("step-ab"); ok {
t.Error("an ask's bound was read as a drop's")
}
if IsAskKey(DropBoundKey("s", "step-ab")) {
t.Error("a drop's bound was taken for an ask's")
}
err := BoundDrop(func(string, string, string, string) error { return nil }, arm, done)(nodebuildgo.Seat, "n1", "step-ab", "why")
if err != nil || len(armed) != 1 || armed[0] != "drop."+nodebuildgo.Seat+".step-ab" || len(answered) != 1 {
t.Errorf("an answered drop: %v, armed %v, answered %v", err, armed, answered)
@@ -0,0 +1,374 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"sort"
"strings"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// TestRuns is the runner's state test-runs (design 47 G §725): one key per test run of a candidate, level and tested
// packages, written create-only by the step that asks the run and changed by compare-and-set.
type TestRuns interface {
// Create writes key when it is absent; ErrExists when it is written already.
Create(key string, r Run) error
// Get reads key and its revision; ErrNoRun when it is absent.
Get(key string) (Run, uint64, error)
// Update writes key at the revision read; ErrChanged when another write came between.
Update(key string, r Run, revision uint64) (uint64, error)
// Delete deletes key at the revision read; ErrChanged when another write came between, a key gone is no fault. No
// write of a run puts a value judged from an earlier read over a later one (the kind of the stale write).
Delete(key string, revision uint64) error
}
// Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by
// whom (A-470d), so the marks are in the key and a restart takes them up.
type Run struct {
seat.TestRun
Readers []RunReader `json:"readers,omitempty"`
}
// RunReader is a step reading a test run, its pipeline key and artifact (§725), marked rerun with the caller of the rerun
// whose new run moves it (A-470d).
type RunReader struct {
Pipeline string `json:"pipeline"`
Artifact string `json:"artifact"`
Rerun bool `json:"rerun,omitempty"`
By string `json:"by,omitempty"`
}
// Of is the reader as the seat names it.
func (r RunReader) Of() seat.TestRunReader {
return seat.TestRunReader{Pipeline: r.Pipeline, Artifact: r.Artifact}
}
// Answer is the run as the seat's read answers it, its readers named by pipeline and artifact.
func (r Run) Answer() seat.TestRun {
t := r.TestRun
t.Readers = nil
for _, o := range r.Readers {
t.Readers = append(t.Readers, o.Of())
}
return t
}
// ErrNoRun is a Get of a test-runs key that is not written.
var ErrNoRun = errors.New("no test run under the key")
// CandidateOf is a pipeline's candidate in A-226's form (§724): `run-<id>` for a run on request, else `<branch>.<revision>`
// of the catalog graph revision it was placed from, built by the node-engine seat's client as the assigner builds it. A
// pipeline naming neither has no candidate, and its steps share no run.
func CandidateOf(p seat.Pipeline) (string, bool) {
var key string
var err error
if p.Run != "" {
key, err = nec.RunKey(p.Run)
} else {
key, err = nec.RevisionKey(p.Branch, p.Revision)
}
return key, err == nil
}
// ShareDigest is a test step's share digest (design 47 G §724, A-469c): the sha256 of the module's build-input fingerprint,
// the level, the step's status in the policy and the step's fetch (its directory and its sorted paths), what the runner
// holds, so two artifacts with one fetch meet and two with different fetches do not.
func ShareDigest(fingerprint, level, status string, fetch nodebuildgo.Fetch) string {
paths := append([]string(nil), fetch.Paths...)
sort.Strings(paths)
h := sha256.New()
for _, part := range []string{fingerprint, level, status, fetch.Dir} {
h.Write([]byte(part + "\n"))
}
h.Write([]byte(strings.Join(paths, "\n")))
return hex.EncodeToString(h.Sum(nil))
}
// TestRunKey is a test run's key in test-runs (§725): `<candidate key>.<level>.<digest>`, the level and the digest each
// escaped as a key's name, the candidate already in its key form.
func TestRunKey(candidate, level, digest string) string {
return candidate + "." + keys.EscapeName(level) + "." + keys.EscapeName(digest)
}
// sharedRun is the one run a step's Shared names, empty when it names none.
func sharedRun(s seat.StepResult) string {
if len(s.Shared) == 0 {
return ""
}
return s.Shared[0].Run
}
// readsRun says whether step s of a pipeline reads run as one step that does not ask it: it names the run and keeps no ask
// of its own (A-469m).
func readsRun(s seat.StepResult, run string) bool {
return run != "" && sharedRun(s) == run && s.Ask == ""
}
// addReaders adds each reader not in r's readers yet, in order.
func addReaders(r *Run, readers []seat.TestRunReader) {
for _, n := range readers {
have := false
for _, o := range r.Readers {
have = have || o.Of() == n
}
if !have {
r.Readers = append(r.Readers, RunReader{Pipeline: n.Pipeline, Artifact: n.Artifact})
}
}
}
// askerOf is the step that asked the run under ask: the step of a pipeline the runner keeps whose own ask it is.
func askerOf(store Updater, ask string) (*seat.SharedAskedBy, error) {
if store == nil || ask == "" {
return nil, nil
}
pkeys, err := store.Keys("*.*.*")
if err != nil {
return nil, fmt.Errorf("the pipelines are not listed: %w", err)
}
for _, key := range pkeys {
p, _, err := store.Get(key)
if err != nil {
continue
}
if i := stepOfAsk(p, ask); i >= 0 {
return &seat.SharedAskedBy{Pipeline: key, Artifact: p.Steps[i].Artifact}, nil
}
}
return nil, nil
}
// share writes the test run of each ask that asks one (§724, §725): the step that writes the key create-only asks the run;
// one that finds the key written by another step reads it, its pipeline's steps of that run then keeping no ask of their
// own and naming the run's ask and its asker (A-469m), and its ask is not put. A run ended `could not run` is never read:
// it is replaced by compare-and-set and asked again. A key written by this very ask (a start asked again) asks it again,
// kept once by its id. It answers the asks still to put.
func (s *Starter) share(key string, asks []Asked) []Asked {
if s.TestRuns == nil {
return asks
}
var out []Asked
joined := map[string]Run{}
for _, a := range asks {
if a.Run == "" {
out = append(out, a)
continue
}
run, ok, err := s.claimRun(a)
var full *fullErr
switch {
case errors.As(err, &full):
// At the cap no run is asked: the steps end could not run, naming the fill (§725, A-469d).
s.logf("the test run %s of %s is not asked: %v", a.Run, key, err)
if s.EndReaders != nil {
if err := s.EndReaders(a.Run, a.Readers, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: err.Error()}); err != nil {
s.logf("the steps of the test run %s, not asked, are not ended: %v", a.Run, err)
}
}
case err != nil:
// A run that cannot be written or read is asked as the step's own, said: slower, never lost.
s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err)
out = append(out, a)
case ok:
out = append(out, a)
default:
joined[a.Run] = run
}
}
if len(joined) > 0 {
s.readOthers(key, joined)
// A run that ended within the candidate is read at once (§724): its readers end with its result.
for run, r := range joined {
if r.Ended == nil || s.EndReaders == nil {
continue
}
var readers []seat.TestRunReader
for _, a := range asks {
if a.Run == run {
readers = a.Readers
}
}
if err := s.EndReaders(run, readers, r.TestRun); err != nil {
s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err)
}
}
}
return out
}
// fullErr is a run not written because test-runs is at its cap (A-469d), naming the fill.
type fullErr struct{ fill seat.Fill }
func (e *fullErr) Error() string {
return fmt.Sprintf("the state %s holds %d of its %d bytes, so no test run is asked (test-runs-cap-mib)", e.fill.Store, e.fill.Bytes, e.fill.Cap)
}
// TestRunsFill is test-runs' fill against its cap setting test-runs-cap-mib, the cap that refuses a new run (A-469d, A-470i),
// never the manifest's ceiling.
func (s *Starter) TestRunsFill() (seat.Fill, error) {
if s.RunsFill == nil {
return seat.Fill{}, errors.New("the runner reads no fill of test-runs")
}
f, err := s.RunsFill()
if err != nil {
return seat.Fill{}, err
}
f.Cap = s.Runs.CapBytes
if f.Cap <= 0 {
f.Cap = DefaultRunSettings.CapBytes
}
return f, nil
}
// atCap is the error of test-runs at its cap setting, nil below it or when its fill is not read (said).
func (s *Starter) atCap() error {
if s.RunsFill == nil {
return nil
}
f, err := s.TestRunsFill()
if err != nil {
s.logf("the fill of test-runs is not read, so its cap is not judged: %v", err)
return nil
}
if f.Bytes >= f.Cap {
return &fullErr{fill: f}
}
return nil
}
// started writes a new run and its untaken bound (§725, A-470).
func (s *Starter) started(a Asked, write func() error) error {
if err := s.atCap(); err != nil {
return err
}
if err := write(); err != nil {
return err
}
if s.Clock != nil {
untaken := s.Runs.Untaken
if untaken <= 0 {
untaken = DefaultRunSettings.Untaken
}
if err := armBound(s.Clock, s.TestRuns, UntakenPrefix+a.Run, a.Run, a.ID, untaken); err != nil {
s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err)
}
}
return nil
}
// claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads.
func (s *Starter) claimRun(a Asked) (Run, bool, error) {
now := s.now()
value := Run{TestRun: seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now}}
addReaders(&value, a.Readers)
for {
err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) })
if err == nil {
return value, true, nil
}
var full *fullErr
if errors.As(err, &full) {
return Run{}, false, err
}
if !errors.Is(err, ErrExists) {
return Run{}, false, err
}
r, rev, err := s.TestRuns.Get(a.Run)
if errors.Is(err, ErrNoRun) {
continue // deleted meanwhile: written again
}
if err != nil {
return Run{}, false, err
}
if r.Ask == a.ID {
return r, true, nil
}
if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun {
err := s.started(a, func() error {
_, err := s.TestRuns.Update(a.Run, value, rev)
return err
})
if errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return Run{}, false, err
}
return value, true, nil
}
addReaders(&r, a.Readers)
if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return Run{}, false, err
}
return r, false, nil
}
}
// readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker
// in Shared (A-469m, A-469q).
func (s *Starter) readOthers(key string, joined map[string]Run) {
if s.Store == nil {
return
}
by := map[string]*seat.SharedAskedBy{}
for run, r := range joined {
asker, err := askerOf(s.Store, r.Ask)
if err != nil {
s.logf("the step that asked the test run %s is not read: %v", run, err)
}
by[run] = asker
}
for {
p, rev, err := s.Store.Get(key)
if err != nil {
s.logf("the pipeline %s is not read, so its steps reading other runs are not written: %v", key, err)
return
}
for i := range p.Steps {
st := &p.Steps[i]
r, ok := joined[sharedRun(*st)]
if !ok || !open(*st) {
continue
}
if r.Node != "" {
st.Outcome, st.On = seat.StepResultOutcomeRunning, "" // its run's ask is taken already (A-469b)
}
st.Ask = ""
st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}}
}
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
s.logf("the pipeline %s's steps reading other runs are not written: %v", key, err)
}
return
}
}
// joinRun adds reader to run's readers by compare-and-set.
func (s *Starter) joinRun(run string, reader seat.TestRunReader) {
if s.TestRuns == nil {
return
}
for {
r, rev, err := s.TestRuns.Get(run)
if err != nil {
s.logf("the test run %s is not read, so %s does not join it: %v", run, reader.Pipeline, err)
return
}
addReaders(&r, []seat.TestRunReader{reader})
if _, err := s.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
s.logf("%s does not join the test run %s: %v", reader.Pipeline, run, err)
}
return
}
}
@@ -0,0 +1,176 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// RunOutput is a test run's output object in test-runs-objects (§725): the packages the run ran with their digests, the
// passing ones it kept and the carried ones it skipped, and the build seat's output.
type RunOutput struct {
Packages []seat.TestRunPackage `json:"packages"`
Output string `json:"output,omitempty"`
}
// Gone is what a run whose output object is gone answers in place of its packages (§725, §733).
const Gone = "the run's output is gone"
// RunsRead answers the seat verb test-runs (design 47 G §728; A-469u) from the runner's state test-runs and its object
// store: each run with its packages read from its output object, and the state's fill.
type RunsRead struct {
Runs Kept
// Object reads an object of test-runs-objects; ErrNoRun when it is gone.
Object func(name string) ([]byte, error)
Fill func() (seat.Fill, error)
}
// TestRuns answers each test run of a candidate, a level, a digest or a module, or every one.
func (r RunsRead) TestRuns(in seat.TestRunsInput) (seat.TestRunsAnswer, error) {
all, err := r.Runs.Keys()
if err != nil {
return seat.TestRunsAnswer{}, fmt.Errorf("the test runs are not listed: %w", err)
}
sort.Strings(all)
a := seat.TestRunsAnswer{Runs: []seat.TestRun{}}
for _, key := range all {
if IsRunBound(key) || (in.Candidate != "" && !strings.HasPrefix(key, in.Candidate+".")) {
continue
}
raw, err := r.Runs.Value(key)
if err != nil || raw == nil {
continue
}
var run Run
if err := json.Unmarshal(raw, &run); err != nil {
continue
}
if (in.Level != "" && run.Level != in.Level) || (in.Digest != "" && run.Digest != in.Digest) || (in.Module != "" && !readsModule(run, in.Module)) {
continue
}
t := run.Answer()
t.Key = key
r.packages(&t)
a.Runs = append(a.Runs, t)
}
if r.Fill != nil {
f, err := r.Fill()
if err != nil {
return seat.TestRunsAnswer{}, fmt.Errorf("the fill of test-runs cannot be read: %w", err)
}
a.Fill = f
}
return a, nil
}
// packages names t's packages from its output object, or says it gone (§725, A-466v).
func (r RunsRead) packages(t *seat.TestRun) {
if t.Output == "" || r.Object == nil {
return
}
data, err := r.Object(t.Output)
if errors.Is(err, ErrNoRun) {
t.Gone = Gone
return
}
var out RunOutput
if err == nil {
err = json.Unmarshal(data, &out)
}
if err != nil {
t.Gone = "the run's output is not read: " + err.Error()
return
}
t.Packages = out.Packages
}
// readsModule says whether a reader of run is a step of module, its artifact `<module>:<name>`.
func readsModule(run Run, module string) bool {
for _, o := range run.Readers {
if strings.HasPrefix(o.Artifact, module+":") {
return true
}
}
return false
}
// TestRunsDrop is the seat verb test-runs-drop (design 47 G §728; A-469g, A-469v), answered by the runner's process: the
// hand act recorded first, the run's ask dropped, each reader ended could not run naming the drop, then the run's output
// object and its key deleted. It answers the run deleted and the readers it ended.
func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.TestRunsDropAnswer, error) {
h := v.Hearer
if h.TestRuns == nil {
return seat.TestRunsDropAnswer{}, errors.New("the runner keeps no test runs")
}
r, _, err := h.TestRuns.Get(in.Key)
if errors.Is(err, ErrNoRun) {
return seat.TestRunsDropAnswer{}, fmt.Errorf("no test run %q in test-runs", in.Key)
}
if err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read: %w", in.Key, err)
}
if err := v.record(seat.VerbTestRunsDrop, "dropped the test run "+in.Key, in.Why); err != nil {
return seat.TestRunsDropAnswer{}, err
}
why := fmt.Sprintf("the test run %s was dropped by %s: %s", in.Key, caller, in.Why)
var ended []seat.TestRunReader
if r.Ended == nil {
if h.DropRun != nil && r.Ask != "" {
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err)
}
}
if err := h.EndRun(in.Key, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil {
return seat.TestRunsDropAnswer{}, err
}
for _, o := range r.Readers {
ended = append(ended, o.Of())
}
}
if h.RunOutputs != nil {
// The run's object is named by its key (endRunOf); one the run named otherwise is deleted too.
for _, name := range []string{in.Key, r.Output} {
if name == "" {
continue
}
if err := h.RunOutputs.Delete(name); err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the output object %s of the test run is not deleted, so its key is kept: %w", name, err)
}
}
}
// The key is deleted at the revision of the run dropped: a run a rerun asked under it since is not deleted with it.
for {
now, rev, err := h.TestRuns.Get(in.Key)
if errors.Is(err, ErrNoRun) {
break
}
if err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read again, so it is not deleted: %w", in.Key, err)
}
if now.Ask != r.Ask {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the key %s holds the run of ask %s, asked since the run of ask %s was dropped: it is not deleted", in.Key, now.Ask, r.Ask)
}
if err := h.TestRuns.Delete(in.Key, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err)
}
break
}
if h.Clock != nil {
for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} {
_ = h.Clock.Unbound(k)
}
}
gone := r.Answer()
gone.Key = in.Key
now := h.now()
if gone.Ended == nil {
gone.Ended, gone.Outcome, gone.Said = &now, seat.StepResultOutcomeCouldNotRun, why
}
return seat.TestRunsDropAnswer{Run: gone, Ended: ended}, nil
}
@@ -0,0 +1,672 @@
package pipeline
import (
"fmt"
"strings"
"testing"
"time"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
)
// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with, the ask it holds
// and its revision.
type clockMem struct {
runs testRunsMem
bound map[string]time.Duration
asks map[string]string
revs map[string]uint64
rev uint64
// before is called once at the next Bound, a write that comes between its caller's read and its write.
before func()
}
func (c *clockMem) Bound(key, ask string, ttl time.Duration) (uint64, error) {
if b := c.before; b != nil {
c.before = nil
b()
}
if c.asks == nil {
c.asks, c.revs = map[string]string{}, map[string]uint64{}
}
c.rev++
c.bound[key], c.asks[key], c.revs[key] = ttl, ask, c.rev
return c.rev, nil
}
func (c *clockMem) Bounded(key string) (string, uint64, bool, error) {
_, ok := c.bound[key]
return c.asks[key], c.revs[key], ok, nil
}
func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil }
func (c *clockMem) UnboundAt(key string, rev uint64) error {
if c.revs[key] == rev {
delete(c.bound, key)
}
return nil
}
func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil }
func (c *clockMem) Keys() ([]string, error) {
var out []string
for k := range c.runs {
out = append(out, k)
}
for k := range c.bound {
out = append(out, k)
}
return out, nil
}
type objectsMem map[string][]byte
func (o objectsMem) Put(name string, data []byte) error { o[name] = data; return nil }
func (o objectsMem) Get(name string) ([]byte, error) {
if d, ok := o[name]; ok {
return d, nil
}
return nil, ErrNoRun
}
func (o objectsMem) Delete(name string) error { delete(o, name); return nil }
// runFixture is a starter and a hearer over one pipelines store and one test-runs, the module's program and tools testing
// one directory.
type runFixture struct {
store memStore
runs testRunsMem
clock *clockMem
objects objectsMem
asks *stepAsks
drops []string
said []seat.StepEvent
logged []string
fill int64
s *Starter
h *Hearer
}
func newRunFixture() *runFixture {
f := &runFixture{store: memStore{}, runs: testRunsMem{}, objects: objectsMem{}, asks: &stepAsks{}}
f.clock = &clockMem{runs: f.runs, bound: map[string]time.Duration{}}
reg := catalog.Registration{Module: "app", Branches: []string{"main"}}
say := func(_ string, body any) error {
if e, ok := body.(seat.StepEvent); ok {
f.said = append(f.said, e)
}
return nil
}
f.h = &Hearer{Store: f.store, Say: say, TestRuns: f.runs, Clock: f.clock, RunOutputs: f.objects,
Logf: func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) },
DropRun: func(seatName, node, ask, why string) error {
f.drops = append(f.drops, seatName+" "+node+" "+ask)
return nil
}}
f.s = &Starter{Pipelines: memCreated{f.store}, Store: f.store, TestRuns: f.runs, Clock: f.clock, Say: say, Steps: f.asks,
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, EndReaders: f.h.EndReaders,
RunsFill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: f.fill}, nil }, Logf: f.h.Logf}
return f
}
func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline {
t.Helper()
entries := map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
}
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
p, _, _ := f.store.Get(Key("main", source, "app"))
return p
}
func (f *runFixture) pipeline(p seat.Pipeline) seat.Pipeline {
q, _, _ := f.store.Get(p.Pipeline)
return q
}
// The test runs' clocks and cap (design 47 G §725; A-466n, A-466p, A-466s, A-468b, A-469d, A-470, A-470a): a run's untaken
// bound is written at its start, its clock at ask-taken and again at each step-progress; a clock's expiry ends the run and
// every reader could not run and drops its ask, a later step-ended ignored and journaled; an untaken bound of a taken run
// ends nothing; a run whose bound passed while the runner was away ends at its start; at the cap no run is asked; an ended
// run is aged with its output object in the step log's order.
func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
t.Run("clock", func(t *testing.T) {
f := newRunFixture()
f.h.Runs = RunSettings{StepBound: 30 * time.Minute, Untaken: 24 * time.Hour}
f.s.Runs = f.h.Runs
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 {
t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound)
}
// A-469b: the run's steps wait for a build node until the holder's ask-taken, then run, each said started again.
if unit.Outcome != seat.StepResultOutcomeWaitingForABuildNode || stepNamed(p, "app:tools", "unit").Outcome != seat.StepResultOutcomeWaitingForABuildNode {
t.Fatalf("the unit steps before ask-taken = %s, %s", unit.Outcome, stepNamed(p, "app:tools", "unit").Outcome)
}
n := len(f.said)
if err := f.h.RunTaken(unit.Ask, "ace"); err != nil {
t.Fatal(err)
}
for _, art := range []string{"app:program", "app:tools"} {
if s := stepNamed(f.pipeline(p), art, "unit"); s.Outcome != seat.StepResultOutcomeRunning {
t.Errorf("the %s unit step after ask-taken is %s, want running", art, s.Outcome)
}
}
if len(f.said)-n != 2 || f.said[n].Result.Outcome != seat.StepResultOutcomeRunning {
t.Errorf("ask-taken said %d step events, want the two unit steps started running", len(f.said)-n)
}
if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute {
t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound)
}
if _, ok := f.clock.bound[UntakenPrefix+run]; ok {
t.Error("the untaken bound of a taken run is kept")
}
// An untaken bound of a taken run ends nothing; progress begins the clock again.
if err := f.h.RunExpired(UntakenPrefix + run); err != nil {
t.Fatal(err)
}
delete(f.clock.bound, BoundPrefix+run)
if err := f.h.RunProgress(unit.Ask); err != nil || f.clock.bound[BoundPrefix+run] != 30*time.Minute {
t.Fatalf("progress = %v, bounds %v: want the clock begun again", err, f.clock.bound)
}
if r, _, _ := f.runs.Get(run); r.Ended != nil {
t.Fatal("a run is ended by its untaken bound after it was taken")
}
// An expiry heard once the bound is armed again ends nothing (A-470o); the bound holds the run's ask (A-470n).
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(run); r.Ended != nil || f.clock.asks[BoundPrefix+run] != unit.Ask {
t.Fatalf("an expiry of a bound armed again ended the run %+v, or the bound holds ask %q", r, f.clock.asks[BoundPrefix+run])
}
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
t.Fatal(err)
}
r, _, _ := f.runs.Get(run)
if r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, unit.Ask) || !strings.Contains(r.Said, "30m") {
t.Errorf("the run whose clock ended = %+v, want could not run naming its ask and the bound", r)
}
if len(f.drops) != 1 || f.drops[0] != "node-build-go ace "+unit.Ask {
t.Errorf("drops = %v, want the run's ask dropped of ace", f.drops)
}
q := f.pipeline(p)
for _, art := range []string{"app:program", "app:tools"} {
if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != r.Said {
t.Errorf("the %s unit step = %+v, want could not run with the run", art, s)
}
}
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", Ended: &ended}); err != nil {
t.Fatal(err)
}
if s := stepNamed(f.pipeline(p), "app:program", "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun {
t.Errorf("a step-ended after the run ended moved the step to %s", s.Outcome)
}
if !strings.Contains(strings.Join(f.logged, "\n"), "the step-ended of ask "+unit.Ask+" is ignored: its test run "+run) {
t.Errorf("the ignored step-ended is not journaled: %q", f.logged)
}
// An untaken run whose bound passed while the runner was away ends at the start.
lint := stepNamed(p, "app:program", "lint").Shared[0].Run
delete(f.clock.bound, UntakenPrefix+lint)
if n, err := f.h.TakeUp(); n != 1 || err != nil {
t.Fatalf("take-up ended %d, %v; want the lint run", n, err)
}
if r, _, _ := f.runs.Get(lint); r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "run-untaken-bound") {
t.Errorf("the lint run taken up = %+v", r)
}
})
t.Run("cap", func(t *testing.T) {
f := newRunFixture()
f.s.Runs.CapBytes, f.fill = 1<<20, 1<<20
p := f.start(t, 4, 9)
for _, a := range f.asks.ids {
if a == stepNamed(p, "app:program", "unit").Ask {
t.Error("a unit run is asked at the cap")
}
}
q := f.pipeline(p)
for _, art := range []string{"app:program", "app:tools"} {
if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "1048576 of its 1048576 bytes") {
t.Errorf("the %s unit step at the cap = %+v, want could not run naming the fill", art, s)
}
}
})
t.Run("age", func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
_ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed, Output: "out-1"}, nil)
f.objects["out-1"] = []byte("ok")
now := agedNow
a := &Ager{Runs: f.runs, RunOutputs: f.objects.Delete, RunsKept: 7 * 24 * time.Hour, Pipelines: agedDeleter{f.store},
Aged: agedState{}, Now: func() time.Time { return now },
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
if from > 1 {
return nil, nil
}
return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools",
What: seat.LogEntryWhatEnded, At: agedNow.Add(-6 * 24 * time.Hour)}}}, nil
}}
a.Age()
if _, _, err := f.runs.Get(run); err != nil {
t.Fatal("a run is aged before test-runs-kept passed")
}
now = now.Add(2 * 24 * time.Hour)
a.Age()
if _, _, err := f.runs.Get(run); err != ErrNoRun || len(f.objects) != 0 {
t.Errorf("the run past test-runs-kept is %v, objects %v; want both deleted", err, f.objects)
}
})
}
// agedDeleter is a memStore with the delete the ager makes.
type agedDeleter struct{ memStore }
func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil }
// How a test run ends and is said (design 47 G §726; A-466i, A-468d, A-469m, A-469z): the run's ask's end ends every step
// reading it with its outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts the run's tests
// once; a step superseded leaves the run to its other readers, and the last to leave drops its ask and ends it could not
// run, no reader left; a composition step's end keeps the runs mesh-wants names.
func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
f := newRunFixture()
a := f.start(t, 4, 9)
b := f.start(t, 6, 9) // the same candidate: it reads a's runs
unit := stepNamed(a, "app:program", "unit")
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed",
Said: "TestX failed", Output: "--- FAIL: TestX", Tests: &nodebuildgo.TestCounts{Unit: 5}, Ended: &ended}); err != nil {
t.Fatal(err)
}
for _, p := range []seat.Pipeline{f.pipeline(a), f.pipeline(b)} {
for _, art := range []string{"app:program", "app:tools"} {
s := stepNamed(p, art, "unit")
if s.Outcome != seat.StepResultOutcomeFailed || s.Why != "TestX failed" || s.Tests == nil || s.Tests.Unit != 5 ||
len(s.Shared) != 1 || s.Shared[0].Run != unit.Shared[0].Run || s.Shared[0].Ask != unit.Ask {
t.Errorf("%s's %s unit step = %+v, want failed with the run's result and shared", p.Pipeline, art, s)
}
}
if c := testCounts(p.Steps); c == nil || c.Unit != 5 {
t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c)
}
}
if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || !strings.Contains(string(f.objects[r.Output]), `"output":"--- FAIL: TestX"`) {
t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects)
}
saidReaders := 0
for _, e := range f.said {
if e.Result.Step == "unit" && e.Result.Outcome == "failed" {
saidReaders++
}
}
if saidReaders != 4 {
t.Errorf("said %d unit step-ended, want one per reading step", saidReaders)
}
// a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends.
lint := stepNamed(a, "app:program", "lint")
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 {
t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops)
}
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil); err != nil {
t.Fatal(err)
}
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask {
t.Errorf("after the last reader left the lint run = %+v, drops %v; want it ended, no reader left, its ask and the integration run's dropped", r, f.drops)
}
// A composition step's end keeps the runs mesh-wants names, as they came (A-468d).
c := newRunFixture()
p := c.start(t, 4, 9)
comp := stepNamed(p, "", "composition")
bs := stepSeatOf(t, "mesh-wants")
raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` +
`"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`)
ev, err := bs.Decode(raw)
if err != nil {
t.Fatal(err)
}
if _, err := c.h.OnStepEndedShared(bs, bs.Prefix+"n1", ev, bs.Shared(raw)); err != nil {
t.Fatal(err)
}
if s := stepNamed(c.pipeline(p), "", "composition"); len(s.Shared) != 1 || s.Shared[0] != (seat.Shared{Machine: "ace", Run: "main.9.ace", Ask: "r1"}) {
t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared)
}
}
// Asking a test run again moves every reader (design 47 G §727; A-469l, A-470d, A-470e): a rerun of one artifact's failed
// step starts a new run under the same key, carrying the former readers, the one that did not pass marked; the new run's
// passing end moves the other artifact's step to passed, marked a rerun by the caller and naming the new ask, and the
// pipeline says pipeline-ended again; a step a person accepted does not move.
func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) {
// takeUp: the rerun's run is ended by the restart's take-up, its bounds gone while the runner was away; a carried reader
// with rerun false, its step passed already, says no step-ended again (A-470l).
for _, c := range []struct{ accepted, takeUp bool }{{false, false}, {true, false}, {false, true}} {
accepted := c.accepted
f := newRunFixture()
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required",
Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}}
f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
p := f.start(t, 4, 9)
e := f.store[p.Pipeline]
e.p.Steps[0].Outcome = seat.StepResultOutcomePassed // the build
f.store[p.Pipeline] = e
unit := stepNamed(p, "app:program", "unit")
end := func(ask string, outcome string, rerun bool) {
t.Helper()
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: outcome,
Rerun: rerun, By: map[bool]string{true: "op@shanks"}[rerun], Ended: &ended}); err != nil {
t.Fatal(err)
}
}
end(unit.Ask, "failed", false)
if q := f.pipeline(p); q.Outcome != seat.PipelineOutcomeFailed || q.EndCount != 1 || stepNamed(q, "app:tools", "unit").Outcome != "failed" {
t.Fatalf("after the failing run the pipeline = %s (%d), tools %s", q.Outcome, q.EndCount, stepNamed(q, "app:tools", "unit").Outcome)
}
if accepted {
e := f.store[p.Pipeline]
i := stepIndex(e.p, "unit", "app:program")
e.p.Steps[i].Outcome, e.p.Steps[i].Accepted = seat.StepResultOutcomeAccepted, &seat.Accepted{By: "op@shanks", Why: "known"}
f.store[p.Pipeline] = e
}
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
return Half{Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
}, Acts: &actLog{}}
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", Why: "flaky"})
if err != nil {
t.Fatal(err)
}
again := a.Result.Ask
if c.takeUp {
run := unit.Shared[0].Run
r, rev, _ := f.runs.Get(run)
r.Readers[0].Rerun, r.Readers[0].By = false, "" // the program's step passed already
if _, err := f.runs.Update(run, r, rev); err != nil {
t.Fatal(err)
}
e := f.store[p.Pipeline]
e.p.Steps[stepIndex(e.p, "unit", "app:program")].Outcome = seat.StepResultOutcomePassed
f.store[p.Pipeline] = e
for k := range f.clock.bound {
delete(f.clock.bound, k)
}
said := len(f.said)
if n, err := f.h.TakeUp(); err != nil || n != 1 {
t.Fatalf("take-up = %d, %v", n, err)
}
for _, ev := range f.said[said:] {
if ev.Result.Artifact == "app:program" {
t.Errorf("a carried reader with rerun false said %s again: %+v", ev.Result.Outcome, ev)
}
}
q := f.pipeline(p)
if stepNamed(q, "app:program", "unit").Outcome != seat.StepResultOutcomePassed || stepNamed(q, "app:tools", "unit").Outcome != seat.StepResultOutcomeCouldNotRun {
t.Errorf("after the take-up program %s, tools %s; want passed kept, tools could not run",
stepNamed(q, "app:program", "unit").Outcome, stepNamed(q, "app:tools", "unit").Outcome)
}
continue
}
r, _, _ := f.runs.Get(unit.Shared[0].Run)
if again == "" || again == unit.Ask || r.Ask != again || r.Ended != nil || len(r.Readers) != 2 || r.Readers[0].Rerun == accepted || (!accepted && r.Readers[0].By != "op@shanks") {
t.Fatalf("the rerun's run = %+v (ask %s), want a new run under the same key carrying the program, marked unless accepted", r, again)
}
end(again, "passed", true)
q := f.pipeline(p)
prog := stepNamed(q, "app:program", "unit")
if accepted {
if prog.Outcome != seat.StepResultOutcomeAccepted {
t.Errorf("an accepted step moved to %s", prog.Outcome)
}
continue
}
if prog.Outcome != seat.StepResultOutcomePassed || !prog.Rerun || prog.By != "op@shanks" || prog.Shared[0].Ask != again {
t.Errorf("the program's step = %+v, want passed, a rerun by the caller, naming the new ask", prog)
}
if q.Outcome != seat.PipelineOutcomePassed || q.EndCount != 2 {
t.Errorf("the pipeline = %s (%d), want passed at its second end", q.Outcome, q.EndCount)
}
}
}
// The test runs' tools (design 47 G §728, §733 unit; A-466v, A-469g, A-469u, A-469v): test-runs answers each run with its
// packages read from its output object, and a run whose object is gone without them, saying so; test-runs-drop records the
// hand act first, ends the readers could not run naming the drop, and deletes the key and its object; with no hand act
// recorded it does nothing.
func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
unit, lint := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:program", "lint")
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed",
Kept: []nodebuildgo.KeptResult{{Package: "app/x", Digest: "d1", Level: "unit"}}, Ended: &ended}); err != nil {
t.Fatal(err)
}
pipes := memKept{}
for k, e := range f.runs {
put(t, pipes, k, e.r)
}
// The fill is said against the cap setting that refuses, never the bucket's 64 MiB ceiling (A-470g, A-470i).
f.s.Runs.CapBytes, f.fill = 1<<20, 9
bucket := f.s.RunsFill
f.s.RunsFill = func() (seat.Fill, error) { fl, err := bucket(); fl.Cap = 64 << 20; return fl, err }
read := RunsRead{Runs: pipes, Object: f.objects.Get, Fill: f.s.TestRunsFill}
a, err := read.TestRuns(seat.TestRunsInput{Level: "unit", Module: "app"})
if err != nil || len(a.Runs) != 1 || len(a.Runs[0].Packages) != 1 || a.Runs[0].Packages[0] != (seat.TestRunPackage{Package: "app/x", Digest: "d1"}) ||
a.Runs[0].Key != unit.Shared[0].Run || len(a.Runs[0].Readers) != 2 || a.Fill.Cap != 1<<20 || a.Fill.Bytes != 9 {
t.Fatalf("test-runs = %+v, %v; want the unit run with its package and readers, and the fill against the cap setting", a, err)
}
delete(f.objects, unit.Shared[0].Run) // gone, whatever step-outputs' age: the run answers without its packages
if a, _ := read.TestRuns(seat.TestRunsInput{Level: "unit"}); len(a.Runs) != 1 || a.Runs[0].Gone != Gone || len(a.Runs[0].Packages) != 0 {
t.Errorf("a run whose output is gone = %+v", a.Runs)
}
acts := &actLog{}
v := Verbs{Hearer: f.h, Starter: f.s, Acts: acts}
if _, err := (Verbs{Hearer: f.h, Starter: f.s}).TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}); err == nil {
t.Fatal("a drop with no hand act recorded was done")
}
f.objects[lint.Shared[0].Run] = []byte("{}")
d, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"})
if err != nil || len(*acts) != 1 || len(d.Ended) != 2 || d.Run.Key != lint.Shared[0].Run || len(f.drops) != 1 {
t.Fatalf("drop = %+v, %v; acts %v, drops %v", d, err, *acts, f.drops)
}
if _, _, err := f.runs.Get(lint.Shared[0].Run); err != ErrNoRun || f.objects[lint.Shared[0].Run] != nil {
t.Errorf("after the drop the key is %v and the object %q, want both gone", err, f.objects[lint.Shared[0].Run])
}
if s := stepNamed(f.pipeline(p), "app:tools", "lint"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "dropped by op@shanks") {
t.Errorf("a reader of the dropped run = %+v", s)
}
}
// racingRuns is test-runs with a write coming between a caller's read and its next update or delete: each hook is called
// once, just before that write.
type racingRuns struct {
testRunsMem
beforeUpdate, beforeDelete func()
}
func (r *racingRuns) Update(key string, v Run, rev uint64) (uint64, error) {
if b := r.beforeUpdate; b != nil {
r.beforeUpdate = nil
b()
}
return r.testRunsMem.Update(key, v, rev)
}
func (r *racingRuns) Delete(key string, rev uint64) error {
if b := r.beforeDelete; b != nil {
r.beforeDelete = nil
b()
}
return r.testRunsMem.Delete(key, rev)
}
// No write of test-runs or its bound keys puts back, ends or deletes a run from a read made before its latest change (design
// 47 G §725, §726, §727, §728; the kind test 109 met): each write is a compare-and-set at the revision read, retried from the
// latest read, and a judgement of a run ends only the run under the ask it judged.
func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
asked := func(f *runFixture, run string) {
t.Helper()
r, rev, _ := f.runs.Get(run)
r.TestRun = seat.TestRun{Level: r.Level, Digest: r.Digest, Seat: r.Seat, Ask: "a2"}
if _, err := f.runs.Update(run, r, rev); err != nil {
t.Fatal(err)
}
}
for _, c := range []struct {
name string
race func(t *testing.T, f *runFixture, p seat.Pipeline, run string)
}{
{"an expiry does not end a run asked again while its ask was dropped", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
unit := stepNamed(p, "app:program", "unit")
_ = f.h.RunTaken(unit.Ask, "ace")
f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil }
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
_ = f.h.RunExpired(BoundPrefix + run)
}},
{"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
f.h.DropRun = func(string, string, string, string) error {
_, _, _ = f.s.claimRun(Asked{Run: run, ID: "a2", Level: "unit", Seat: "node-build-go",
Readers: []seat.TestRunReader{{Pipeline: "main.5.app", Artifact: "app:program"}}})
return nil
}
for _, a := range []string{"app:program", "app:tools"} {
f.h.leaveRun(p.Pipeline, stepNamed(p, a, "unit"), "cancelled")
}
}},
{"the age does not delete a run asked again since it was read", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
_ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil)
runs := &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
a := &Ager{Runs: runs, RunsKept: time.Hour, Pipelines: agedDeleter{f.store}, Aged: agedState{}, Now: func() time.Time { return agedNow },
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
if from > 1 {
return nil, nil
}
return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools",
What: seat.LogEntryWhatEnded, At: agedNow.Add(-2 * time.Hour)}}}, nil
}}
a.Age()
}},
{"test-runs-drop does not delete a run asked again under its key", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
v := Verbs{Hearer: f.h, Starter: f.s, Acts: &actLog{}}
if _, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: run, Why: "stuck"}); err == nil {
t.Error("a drop whose key holds a run asked since answers no error")
}
}},
{"a step-progress heard as the run ends leaves no clock", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
unit := stepNamed(p, "app:program", "unit")
_ = f.h.RunTaken(unit.Ask, "ace")
f.clock.before = func() { _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) }
_ = f.h.RunProgress(unit.Ask)
if _, ok := f.clock.bound[BoundPrefix+run]; ok {
t.Error("a clock written from a read before the run's end is kept")
}
asked(f, run) // the key's run is asked again, so the check below holds
}},
{"a clock of a run not taken ends nothing", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
asked(f, run)
_ = f.h.RunExpired(BoundPrefix + run)
}},
} {
t.Run(c.name, func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
run := stepNamed(p, "app:program", "unit").Shared[0].Run
c.race(t, f, p, run)
if r, _, err := f.runs.Get(run); err != nil || r.Ask != "a2" || r.Ended != nil {
t.Errorf("the key's run = %+v, %v; want the run asked again under a2, running", r, err)
}
})
}
// A late signal of a replaced run arms no bound and is said in the journal (A-470n).
t.Run("a late ask-taken or step-progress of a replaced run moves nothing", func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
asked(f, run)
delete(f.clock.bound, UntakenPrefix+run)
_ = f.h.RunTaken(unit.Ask, "ace")
_ = f.h.RunProgress(unit.Ask)
late := 0
for _, l := range f.logged {
if strings.Contains(l, unit.Ask) && strings.Contains(l, "moves nothing") {
late++
}
}
if _, ok := f.clock.bound[BoundPrefix+run]; ok || late != 2 {
t.Errorf("bounds %v, journal %q; want no clock and both late signals journaled", f.clock.bound, f.logged)
}
})
// A run's end is written as its step-ended is heard, before the finding the step says (A-470o).
t.Run("the run ends before the step's finding is written", func(t *testing.T) {
f := newRunFixture()
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required",
Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}}
f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
endedFirst := false
f.h.Failures = &Failures{Findings: newHeld(), Author: func(string, string) (string, error) {
r, _, _ := f.runs.Get(run)
endedFirst = r.Ended != nil
return "alice", nil
}}
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
t.Fatal(err)
}
if !endedFirst {
t.Error("the step's finding was written before its run's end")
}
})
// A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read,
// carrying that reader, never answering the change as an error nor putting back the earlier readers.
t.Run("a rerun writes its run from the latest read", func(t *testing.T) {
f := newRunFixture()
p := f.start(t, 4, 9)
unit := stepNamed(p, "app:program", "unit")
run := unit.Shared[0].Run
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
t.Fatal(err)
}
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeUpdate: func() {
r, rev, _ := f.runs.Get(run)
r.Readers = append(r.Readers, RunReader{Pipeline: "main.9.app", Artifact: "app:program"})
_, _ = f.runs.Update(run, r, rev)
}}
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
return Half{Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
}, Acts: &actLog{}}
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"})
if err != nil {
t.Fatalf("a rerun whose read was changed before its write: %v", err)
}
r, _, _ := f.runs.Get(run)
joined := false
for _, o := range r.Readers {
joined = joined || o.Pipeline == "main.9.app"
}
if r.Ask != a.Result.Ask || r.Ended != nil || !joined || len(r.Readers) != 3 {
t.Errorf("the rerun's run = %+v; want the new ask running, carrying the reader that joined", r)
}
})
}
@@ -27,8 +27,8 @@ type Verbs struct {
Candidate func(group, branch string, revision int64) (Half, error)
// Acts records a hand act before the verb does it (rerun, accept, cancel); a nil Acts refuses the verb, nothing recorded.
Acts views.HandActs
// Drop drops the bound of an ask a cancelled pipeline no longer waits on.
Drop func(ask string) error
// Runs answers the read verb test-runs, answered by the process that holds test-runs-cap-mib (A-470i).
Runs RunsRead
}
func (v Verbs) now() time.Time { return v.Hearer.now() }
@@ -185,8 +185,8 @@ func (v Verbs) Accept(caller string, in seat.AcceptInput) (seat.AcceptAnswer, er
return seat.AcceptAnswer{Result: p.Steps[i]}, nil
}
// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and the bound of each ask dropped, recorded in
// the hand-act log first (§394); it says pipeline-ended.
// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and each step of a test run leaving it, recorded
// in the hand-act log first (§394, §726); it says pipeline-ended.
func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, error) {
h := v.Hearer
recorded := false
@@ -205,14 +205,14 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
recorded = true
}
now := h.now()
var drop []string
var leave []seat.StepResult
for i := range p.Steps {
s := &p.Steps[i]
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if s.Ask != "" {
drop = append(drop, s.Ask)
if sharedRun(*s) != "" && s.Seat != wants.Seat {
leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726)
}
s.Outcome, s.By, s.Why, s.Ended = seat.StepResultOutcomeCancelled, caller, in.Why, &now
}
@@ -223,12 +223,8 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
} else if err != nil {
return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err)
}
if v.Drop != nil {
for _, ask := range drop {
if err := v.Drop(ask); err != nil {
h.logf("the bound of ask %s of the cancelled %s is not dropped: %v", ask, in.Pipeline, err)
}
}
for _, s := range leave {
h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why)
}
return seat.CancelAnswer{Pipeline: p}, h.said(p, false, true)
}
@@ -242,6 +238,11 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
if in.Step == seat.StepResultStepApproval {
return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun")
}
if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil {
if i, err := stepOf(p, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) {
return v.rerunTest(caller, in, p.Steps[i])
}
}
var ask *Asked
p, _, err := v.change(in.Pipeline, seat.EventStepStarted, func() error {
return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why)
@@ -289,7 +290,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449)
if ask != nil {
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
}
}
@@ -301,6 +302,173 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &r}, nil
}
// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one
// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set
// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass.
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) {
h := v.Hearer
run := sharedRun(st)
if open(st) {
return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline)
}
r, rev, err := h.TestRuns.Get(run)
if err != nil && !errors.Is(err, ErrNoRun) {
return seat.RerunAnswer{}, fmt.Errorf("the test run %s is not read: %w", run, err)
}
exists := err == nil
reader := seat.TestRunReader{Pipeline: in.Pipeline, Artifact: st.Artifact}
read := exists && r.Ended != nil && r.Outcome != seat.StepResultOutcomeCouldNotRun && st.Ended != nil && r.Started != nil && r.Started.After(*st.Ended)
join := exists && r.Ended == nil
var ask *Asked
if !join && !read {
p, _, err := h.Store.Get(in.Pipeline)
if err != nil {
return seat.RerunAnswer{}, err
}
if ask, err = v.askOf(p, st, caller, h.now()); err != nil {
return seat.RerunAnswer{}, err
}
}
var asker *seat.SharedAskedBy
if join || read {
if asker, err = askerOf(h.Store, r.Ask); err != nil {
return seat.RerunAnswer{}, err
}
if asker != nil && *asker == (seat.SharedAskedBy{Pipeline: reader.Pipeline, Artifact: reader.Artifact}) {
asker = nil
}
}
event := seat.EventStepStarted
if read {
event = seat.EventStepEnded
}
p, _, err := v.change(in.Pipeline, event, func() error {
return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why)
}, func(p *seat.Pipeline, now time.Time) (int, error) {
switch p.Outcome {
case seat.PipelineOutcomeCancelled, seat.PipelineOutcomeSuperseded:
return -1, fmt.Errorf("the pipeline %s is %s: a step of it is not asked again", p.Pipeline, p.Outcome)
}
i, err := stepOf(*p, in.Step, in.Artifact)
if err != nil {
return -1, err
}
s := &p.Steps[i]
*s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
Outcome: seat.StepResultOutcomeRunning, By: caller, Rerun: true, Started: &now}
switch {
case ask != nil:
s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}}
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode // until ask-taken (A-469b)
case read:
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now
if r.Tests != nil {
t := *r.Tests
s.Tests = &t
}
default:
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
if r.Node == "" {
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
}
}
if p.Outcome != seat.PipelineOutcomeRunning {
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
}
for j := range p.Steps {
if p.Steps[j].Step == seat.StepResultStepApproval && p.Steps[j].Outcome == seat.StepResultOutcomeNotAsked {
p.Steps[j].Outcome, p.Steps[j].Why = seat.StepResultOutcomeWaiting, ""
}
}
return i, nil
})
if err != nil {
return seat.RerunAnswer{}, err
}
h.pin(p)
switch {
case join || read:
v.Starter.joinRun(run, reader)
default:
now := h.now()
a := *ask
a.Run = run
// The new run is written from the latest read of the key: a write between (a reader leaving, a join) is read again and
// its readers carried, never put back; a run another step started since is said, not replaced.
for {
value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}}
// Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d).
for _, o := range r.Readers {
if o.Of() == reader {
continue
}
o.Rerun, o.By = !v.passedReader(o, run), ""
if o.Rerun {
o.By = caller
}
value.Readers = append(value.Readers, o)
}
addReaders(&value, []seat.TestRunReader{reader})
err := v.Starter.started(a, func() error {
if !exists {
return h.TestRuns.Create(run, value)
}
_, err := h.TestRuns.Update(run, value, rev)
return err
})
if errors.Is(err, ErrChanged) || errors.Is(err, ErrExists) {
got, gotRev, gerr := h.TestRuns.Get(run)
switch {
case errors.Is(gerr, ErrNoRun):
r, rev, exists = Run{TestRun: seat.TestRun{Digest: r.Digest}}, 0, false
continue
case gerr != nil:
err = gerr
case got.Ended == nil:
err = fmt.Errorf("the run of ask %s was asked under the key since it was read", got.Ask)
default:
r, rev, exists = got, gotRev, true
continue
}
}
if err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err)
}
break
}
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
}
}
i, err := stepOf(p, in.Step, in.Artifact)
if err != nil {
return seat.RerunAnswer{}, err
}
res := p.Steps[i]
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil
}
// passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run.
func (v Verbs) passedReader(reader RunReader, run string) bool {
p, _, err := v.Hearer.Store.Get(reader.Pipeline)
if err != nil {
return true // a pipeline gone has no step to move
}
for _, s := range p.Steps {
if s.Artifact == reader.Artifact && sharedRun(s) == run {
switch s.Outcome {
case seat.StepResultOutcomePassed, seat.StepResultOutcomeCarried, seat.StepResultOutcomeAccepted,
seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeCancelled:
return true
}
return s.Accepted != nil
}
}
return true
}
// askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice
// under one id is kept once), saying it is a rerun and by whom.
func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) {
@@ -325,7 +493,7 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
} else {
return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch)
}
_, placed := Place(p, reg, half.Entries, now, v.Starter.Name)
_, placed := place(p, reg, half.Entries, now, v.Starter.Name, false)
base := AskID(startOf(p, now), p.Pipeline, s.Artifact, s.Step)
for _, a := range placed {
if a.ID != base {
@@ -35,8 +35,9 @@ func verbsFixture() (Verbs, memStore, *[]string, *stepAsks, *actLog, *[]string)
asks, acts, dropped := &stepAsks{}, &actLog{}, &[]string{}
s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true },
Now: h.Now}
return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts,
Drop: func(ask string) error { *dropped = append(*dropped, ask); return nil }}, store, said, asks, acts, dropped
h.DropRun = func(_, _, ask, _ string) error { *dropped = append(*dropped, ask); return nil }
return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts},
store, said, asks, acts, dropped
}
func stepNamed(p seat.Pipeline, artifact, step string) seat.StepResult {
@@ -81,7 +82,8 @@ func TestReviewEndsThePlacedReviewStepWithItsProvenCaller(t *testing.T) {
// accept takes only a failed optional step, writes its hand act first, and refuses without one; rerun asks the step again
// under a new ask saying it is a rerun and its caller, reopens a failed pipeline and asks nothing of a refused one; cancel ends
// a running pipeline cancelled, each step cancelled and each ask's bound dropped (design 47 G §22, §30, §325, §394).
// a running pipeline cancelled, each step cancelled, no ask's bound left to drop (design 47 G §22, §30, §325, §394; §647 as
// A-466y amends it).
func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testing.T) {
v, store, said, asks, acts, dropped := verbsFixture()
end := func(step, outcome string) {
@@ -134,7 +136,7 @@ func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testin
}
// A cancel ends the pipeline cancelled and drops the bounds of the asks still held.
c, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "wrong branch"})
if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) == 0 {
if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) != 0 {
t.Fatalf("cancel = %+v, %v, dropped %v", c.Pipeline, err, *dropped)
}
if s := stepNamed(c.Pipeline, "app:program", "lint"); s.Outcome != "cancelled" || s.By != "operator" {
@@ -55,9 +55,12 @@ func TestManifest(t *testing.T) {
{"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat &&
equalSets(m.Claims[0].Serves, seat.Verbs())},
// §27, §85, §330, §355, §77: the old state sections.
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "toolchains", "findings", "first-views", "consumers"})},
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})},
// §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live.
{"bounds is key-ttl", keyTTL["bounds"]},
// §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling.
{"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64},
{"the test runs' settings", runSettingsOf(raw)},
// RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB.
{"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256},
{"kept 128 MiB", stateMiB[seat.StateKept] == 128},
@@ -78,10 +81,13 @@ func TestManifest(t *testing.T) {
{"consumes", equalSets(m.Consumes, []string{
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken",
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})},
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed",
// §647 as A-466z amends it: no holder is read from the seat-holders view.
"seat:mesh-module-assigner.views.seat-holders"})},
// RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and
// asked again by hand (a hand act).
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations",
@@ -107,7 +113,8 @@ func TestManifest(t *testing.T) {
"seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})},
{"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})},
// RN25 (§539): the runner's tools carry the prefix runner_.
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})},
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append",
"runner_test_runs", "runner_test_runs_drop"})},
// A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery,
// never by a tool of its own; runner_health stays a tool.
{"health is the unit", healthTool(raw) == "unit"},
@@ -175,3 +182,43 @@ func equalSets(a, b []string) bool {
slices.Sort(y)
return slices.Equal(x, y)
}
// runSettingsOf says whether the manifest holds the test runs' four settings (design 47 G §725; A-469w, A-469x): the cap a
// whole number of at most 64, the three times Go durations, each written into the file its environment names.
func runSettingsOf(raw []byte) bool {
var m struct {
Config map[string]map[string]any `json:"config"`
Resources []struct {
Type string `json:"type"`
Path string `json:"path"`
Content string `json:"content"`
Env map[string]string `json:"env"`
} `json:"resources"`
}
if json.Unmarshal(raw, &m) != nil {
return false
}
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string"}
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) {
return false
}
env := map[string]bool{}
for _, r := range m.Resources {
for _, v := range r.Env {
env[v] = true
}
}
for key, kind := range want {
if m.Config[key]["type"] != kind {
return false
}
found := false
for _, r := range m.Resources {
found = found || (r.Type == "file" && r.Content == "${config:"+key+"}" && env[r.Path])
}
if !found {
return false
}
}
return true
}
+82 -4
View File
@@ -18,7 +18,9 @@
"rerun",
"accept",
"cancel",
"ask-again"
"ask-again",
"test-runs",
"test-runs-drop"
]
}
],
@@ -27,7 +29,9 @@
"runner_first_view_again",
"runner_health",
"runner_unlogged",
"runner_unlogged_append"
"runner_unlogged_append",
"runner_test_runs",
"runner_test_runs_drop"
],
"consumes": [
"mesh-delivery-indexer.take-in.taken",
@@ -41,13 +45,17 @@
"node-build-go.ask-taken",
"node-build-node.ask-taken",
"node-build-image.ask-taken",
"node-build-go.step-progress",
"node-build-node.step-progress",
"node-build-image.step-progress",
"mesh-wants.step-ended",
"mesh-catalog.registration-changed",
"mesh-catalog.catalog-revision",
"mesh-conditions.findings-asked",
"mesh-delivery.recovery-merged",
"mesh-catalog.seat-definition-changed",
"mesh-catalog.seat-definition-removed"
"mesh-catalog.seat-definition-removed",
"seat:mesh-module-assigner.views.seat-holders"
],
"invokes": [
"seat:mesh-catalog.seat-definitions",
@@ -79,6 +87,34 @@
"uses": [
"operator-channel"
],
"config": {
"test-runs-cap-mib": {
"type": "number",
"description": "MiB of the state test-runs at which the runner asks no new test run and ends the step could not run, naming the fill (design 47 G section 725, A-469d); the manifest's max-mib is the ceiling.",
"minimum": 1,
"maximum": 64,
"trusted": false,
"default": 16
},
"test-runs-kept": {
"type": "string",
"description": "how long an ended test run and its output object are kept after its end, a Go duration such as 168h (design 47 G section 725, A-469x, A-470a); at most 168h, so the pipeline naming its key is still kept.",
"trusted": false,
"default": "168h"
},
"test-step-bound": {
"type": "string",
"description": "how long a taken test run's ask may say no progress before the run ends could not run, a Go duration such as 30m, counted from the build seat's ask-taken and again from each step-progress (design 47 G section 725, A-466n, A-469x).",
"trusted": false,
"default": "30m"
},
"run-untaken-bound": {
"type": "string",
"description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).",
"trusted": false,
"default": "24h"
}
},
"state": [
{
"name": "pipelines",
@@ -107,6 +143,12 @@
"history": 1,
"key-ttl": true
},
{
"name": "test-runs",
"history": 1,
"max-mib": 64,
"key-ttl": true
},
{
"name": "findings",
"history": 1
@@ -162,6 +204,38 @@
"place": ".",
"unknown": "program"
},
{
"id": "test-runs-cap",
"type": "file",
"path": "${dir:state}/test-runs-cap-mib",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:test-runs-cap-mib}"
},
{
"id": "test-runs-kept",
"type": "file",
"path": "${dir:state}/test-runs-kept",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:test-runs-kept}"
},
{
"id": "test-step-bound",
"type": "file",
"path": "${dir:state}/test-step-bound",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:test-step-bound}"
},
{
"id": "run-untaken-bound",
"type": "file",
"path": "${dir:state}/run-untaken-bound",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:run-untaken-bound}"
},
{
"id": "pipeline",
"type": "process",
@@ -171,7 +245,11 @@
"./mesh-delivery-pipeline"
],
"env": {
"MESH_BROKER_FILE": "${dir:state}/broker"
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_RUNNER_TEST_RUNS_CAP_FILE": "${dir:state}/test-runs-cap-mib",
"MESH_RUNNER_TEST_RUNS_KEPT_FILE": "${dir:state}/test-runs-kept",
"MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound",
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound"
},
"user": "mesh-delivery-pipeline",
"health": {
@@ -67,6 +67,11 @@ type Request struct {
// them (the definitions of the seat mesh-catalog, resolved), raw so that this package names no seat (ADR 0635 §4, ADR 0636
// §1). The judge and the composer read them. None is no list at hand: a composer handed none judges no machine (ADR 0636 §1).
Seats []json.RawMessage `json:"seats,omitempty"`
// BlameTries bounds the composer's search for the modules a machine that does not compose blames: it composes at
// most this many times more, the assigner's setting blame-tries, since the composer has no settings of its own
// (design 47 G §721, §729; A-469i). None is no search: the composer blames only what its composition holds or
// refuses (A-468i).
BlameTries int `json:"blame_tries,omitempty"`
}
// Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs.
@@ -85,6 +90,11 @@ type Case struct {
Verdicts map[string]json.RawMessage `json:"verdicts,omitempty"`
Kept map[string]map[string]Manifest `json:"kept,omitempty"`
Assignments map[string]json.RawMessage `json:"assignments,omitempty"`
// None is, by module, the words of the revision's none for each module of the case the revision gives no fingerprint
// (its build can never be made: builders disagree, no build module), the same word as the revision's. The assigner
// fills it from the revision it reads for the run; the composer reads it alone to tell a build that can never be made
// (design 47 G A-470j, A-470k).
None map[string]string `json:"none,omitempty"`
}
// Manifest is a module's manifest as its repository holds it (Raw, the bytes), or the digest the delivery indexer keeps
@@ -168,6 +178,11 @@ type ManifestRead struct {
// in words, why the composer did not judge the machine (an older node-engine's input that does not say how it composes,
// novox/hq ADR 0593 §9): the machine is then neither composed nor refused, and a reader decides by this field alone,
// never by the words of Reason.
//
// Blamed names each module the candidate's composition of the machine holds or refuses and the composition without the
// candidate does not, with its problem (design 47 G §721). A composer of this shape always writes it, an empty list when
// it blames none (A-469h); nil is a verdict with no such field, a composer from before §721, which its reader judges as
// before it ("the composer names no culprits").
type Machine struct {
Composes bool `json:"composes"`
NotJudged string `json:"not_judged,omitempty"`
@@ -175,6 +190,10 @@ type Machine struct {
Problems []string `json:"problems,omitempty"`
Refusals []string `json:"refusals,omitempty"`
Declared []Path `json:"declared,omitempty"`
Blamed []struct {
Module string `json:"module"`
Problem string `json:"problem"`
} `json:"blamed,omitzero"`
}
// Path is one path a declaration puts on the machine, with its kind (the resource's type), owner, group and mode as
@@ -57,6 +57,51 @@ func TestAManifestIsItsBytesOrItsDigest(t *testing.T) {
t.Errorf("the known packages %#v went as %s and read back %#v (%v)", known, wire, back.Packages, err)
}
}
// K1/AC4 (design 47 G §721, §729; A-469h, A-469i): a request's blame_tries and a verdict's blamed read back whole; a
// verdict with no blamed (a composer from before §721) reads back as no field, told apart from an empty list.
wire, err = json.Marshal(Request{V: Version, BlameTries: 20})
if back, derr := Decode(wire); err != nil || derr != nil || back.BlameTries != 20 || !strings.Contains(string(wire), `"blame_tries":20`) {
t.Errorf("blame_tries 20 went as %s and read back %d (%v, %v)", wire, back.BlameTries, err, derr)
}
// H3 (design 47 G A-470j, A-470k): a case's none, each module the revision gives no fingerprint with its none words,
// reads back whole; a case naming none leaves the field off the wire.
wire, err = json.Marshal(Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{},
None: map[string]string{"lidarr": "builders disagree: node"}}, {Name: "trunk", Manifests: map[string]Manifest{}}}})
if back, derr := Decode(wire); err != nil || derr != nil || back.Cases[0].None["lidarr"] != "builders disagree: node" ||
back.Cases[1].None != nil || !strings.Contains(string(wire), `"none":{"lidarr":"builders disagree: node"}`) ||
strings.Count(string(wire), `"none"`) != 1 {
t.Errorf("a case's none went as %s and read back %+v (%v, %v)", wire, back.Cases, err, derr)
}
one := Machine{}
one.Blamed = append(one.Blamed, struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{Module: "a", Problem: "holds /etc/x"})
for _, c := range []struct {
name string
wire string
blamed int
old bool
}{
{"blames one", `{"composes":false,"blamed":[{"module":"a","problem":"holds /etc/x"}]}`, 1, false},
{"blames none", `{"composes":false,"blamed":[]}`, 0, false},
{"an old composer", `{"composes":false}`, 0, true},
} {
var m Machine
if err := json.Unmarshal([]byte(c.wire), &m); err != nil {
t.Fatalf("%s: %v", c.name, err)
}
if len(m.Blamed) != c.blamed || (m.Blamed == nil) != c.old {
t.Errorf("%s: %s read back blamed %#v", c.name, c.wire, m.Blamed)
}
again, err := json.Marshal(m)
if err != nil || string(again) != c.wire {
t.Errorf("%s: %s went again as %s (%v)", c.name, c.wire, again, err)
}
}
if wire, _ := json.Marshal(one); string(wire) != `{"composes":false,"blamed":[{"module":"a","problem":"holds /etc/x"}]}` {
t.Errorf("a machine blaming a went as %s", wire)
}
bad := Request{V: Version, Cases: []Case{{Name: "c", Manifests: map[string]Manifest{"x": {Digest: "nope"}}}}}
if err := Fill(&bad, func(Manifest) ([]byte, error) { return nil, errors.New("not there") }); err == nil || !strings.Contains(err.Error(), "nope") {
t.Fatalf("a digest the indexer does not hold: %v", err)
@@ -190,17 +190,23 @@ func brokerSubject(subject string) bool {
return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.")
}
// pipelineSubject hands the gate's `source-moved` and the runner's `pipeline-started` to the watchman (watch/pipelines.go,
// design 47 G §12): the head of a watched branch and its pipeline. It says whether the subject was one of them.
// pipelineSubject hands the gate's `source-moved`, the runner's `pipeline-started`, `step-started` and `step-ended` and the assigner's
// `step-progress` to the watchman (watch/pipelines.go, watch/steps.go, design 47 G §12, §732): the head of a watched branch, its
// pipeline and each step's clock. It says whether the subject was one of them.
func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool {
switch {
case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."):
w.SourceMoved(body)
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."):
w.PipelineStarted(body)
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."),
strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."):
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."):
w.StepEvent(body)
w.StepStarted(body)
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."):
w.StepEvent(body)
w.StepEnded(body)
case strings.HasPrefix(subject, "mesh.seat."+meshwants.Seat+".event."+meshwants.EventStepProgress+"."):
w.WantsStepProgress(body)
default:
return false
}
@@ -22,6 +22,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscompose"
@@ -331,6 +332,55 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
return finding(watch.FindingKey(watch.ScopeDelivery, stuckOf(askStream, ask1), watch.KindStuck))
})
until("node n1's silence", func() bool { return finding("machine.n1.silent") })
// Design 47 G §732 (test 108's watchman case): a composition step whose clock passed five minutes with no progress is
// `pipeline-stands`; the assigner's `step-progress` for its ask clears it, and the step's own `step-ended` ends it.
const pipeline, stepAsk = "main.9.web", "wants-ask-1"
stepKey := pipeline + "/composition/"
stepFinding := watch.FindingKey(watch.ScopeDelivery, watch.StepFindingID(stepKey), watch.KindPipelineStands)
sayStep := func(event string, at time.Time) {
t.Helper()
body, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "web", Commit: "c1", At: &at,
Result: meshpipelinerunner.StepResult{Step: "composition", Ask: stepAsk, Outcome: meshpipelinerunner.StepResultOutcomeRunning}})
if _, err := js.Publish(ctx, "mesh.seat.mesh-pipeline-runner.event."+event+".web.main.c1", body); err != nil {
t.Fatal(err)
}
}
sayStep("step-started", time.Now().UTC().Add(-6*time.Minute))
until("the composition step with no progress for six minutes", func() bool { return finding(stepFinding) })
progress, _ := json.Marshal(meshwants.StepProgressEvent{Ask: stepAsk, Asker: "runner", At: time.Now().UTC(), Phase: "run"})
if _, err := js.Publish(ctx, "mesh.seat.mesh-wants.event.step-progress.n1", progress); err != nil {
t.Fatal(err)
}
until("the step's progress clears it", func() bool { return !finding(stepFinding) })
sayStep("step-started", time.Now().UTC().Add(-6*time.Minute))
until("the step stands again", func() bool { return finding(stepFinding) })
sayStep("step-ended", time.Now().UTC())
until("the step's end clears it", func() bool { return !finding(stepFinding) })
// A step the runner's stands named at the start (seeded through the seat verb) and that no progress follows is
// `pipeline-stands`; the approval waiting on a person is owed none.
seeded := func(step string) string {
return watch.FindingKey(watch.ScopeDelivery, watch.StepFindingID(seededPipeline+"/"+step+"/"), watch.KindPipelineStands)
}
until("the seeded composition step with no progress", func() bool { return finding(seeded("composition")) })
if finding(seeded("approval")) {
t.Error("a step waiting on a person raised pipeline-stands")
}
// A test step whose run's ask ended with no step-ended of its own stands five minutes (here the short bound) after that end.
const runAsk = "run-ask-2"
old := time.Now().UTC().Add(-time.Minute) // started longer ago than the short bound: it stands at once
testStep, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: "main.6.web", Branch: "main", Module: "web", Commit: "c2", At: &old,
Result: meshpipelinerunner.StepResult{Step: "unit", Artifact: "web:program", Ask: "t-ask", Outcome: meshpipelinerunner.StepResultOutcomeRunning,
Shared: []meshpipelinerunner.Shared{{Run: "r2", Ask: runAsk}}}})
if _, err := js.Publish(ctx, "mesh.seat.mesh-pipeline-runner.event.step-started.web.main.c2", testStep); err != nil {
t.Fatal(err)
}
testFinding := watch.FindingKey(watch.ScopeDelivery, watch.StepFindingID("main.6.web/unit/web:program"), watch.KindPipelineStands)
until("the test step standing since its old start", func() bool { return finding(testFinding) })
sayAsk(t, ctx, js, "step-ended", runAsk)
until("the end of the run's ask begins the clock again, so the finding is cleared", func() bool { return !finding(testFinding) })
until("the test step whose run ended with no end of its own, one bound after that end", func() bool { return finding(testFinding) })
// A-274: the finding is said under the watchman's binding finding-events.
until("a finding-written under the binding finding-events", func() bool {
st, err := js.Stream(ctx, "WATCHMAN_FINDINGS")
@@ -605,6 +655,9 @@ const (
testWantsBucket = testAssigner + "_" + meshwants.StateWants
)
// seededPipeline is the pipeline the stand-in runner's stands names.
const seededPipeline = "main.5.web"
// serveCatalogHalf answers the catalog half's first views (design 47 G §614): the gate's one source branch, the stand-in
// registered on it, and its trunk revision whose entry claims mesh-module-assigner.
func serveCatalogHalf(t *testing.T, nc *nats.Conn) {
@@ -617,7 +670,13 @@ func serveCatalogHalf(t *testing.T, nc *nats.Conn) {
Entries: []meshcatalog.CatalogEntry{{Module: testAssigner, ManifestDigest: "d1", Manifest: map[string]any{"module": testAssigner,
"claims": []any{map[string]any{"name": "mesh-module-assigner", "scope": "mesh"}}}}}}
registrations := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{{Module: testAssigner, Version: 1, Branches: []string{"release"}}}}
for subject, handle := range map[string]nats.MsgHandler{"mesh.seat.mesh-catalog.call.revision": respond(revision),
// The runner's stands (design 47 G §731, A-469p), which the watchman seeds its open steps from at its start: a composition step
// waiting on the assigner, a test step reading a run, and an approval waiting on a person.
stands := meshpipelinerunner.StandsAnswer{Stands: []meshpipelinerunner.Wait{
{Pipeline: seededPipeline, Step: "composition", Ask: "seeded-ask", Module: "web", Branch: "main", Since: time.Now().UTC(), V: 1},
{Pipeline: seededPipeline, Step: "approval", On: meshpipelinerunner.WaitOnPerson, Module: "web", Branch: "main", Since: time.Now().UTC(), V: 1}}}
for subject, handle := range map[string]nats.MsgHandler{"mesh.seat.mesh-pipeline-runner.call.stands": respond(stands),
"mesh.seat.mesh-catalog.call.revision": respond(revision),
"mesh.seat.mesh-catalog.call.registrations": respond(registrations), "mesh.seat.mesh-delivery.call.source": respond(map[string]any{"branches": []string{"release"}})} {
for _, s := range []string{subject, subject + ".>"} {
sub, err := nc.Subscribe(s, handle)
@@ -13,6 +13,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"io"
"log"
@@ -400,6 +401,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
atCatalog := atHolder(calls.Call, holders, meshcatalog.Seat) // the indexer holds the seat mesh-catalog
seedTakeIn(w, atCatalog, logger.Printf)
seedVerdicts(w, atCatalog, logger.Printf)
seedSteps(w, calls.Call, logger.Printf)
calls.Close()
}
// The take-in events are the indexer's ordinary events (its emits), heard on the shared events stream through this
@@ -507,6 +509,24 @@ func seedTakeIn(w *watch.Watchman, call func(key string, body any) (json.RawMess
logf("take-in: %s cannot be asked (%v), so a wait the indexer held before this start is said only once a take-in event of it is heard", pendingTool, err)
}
// standsVerb is the runner's seat verb that answers every step started and not ended (design 47 G §731, A-469p).
const standsVerb = "seat:" + meshpipelinerunner.Seat + "." + meshpipelinerunner.VerbStands
// seedSteps asks the runner which steps stand now and keeps each, its clock begun at this start (Watchman.SeedSteps, design 47 G
// §731, §732). A runner that does not answer is said: the steps are then known from the events heard after.
func seedSteps(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) {
raw, err := call(standsVerb, map[string]any{})
if err == nil {
var a meshpipelinerunner.StandsAnswer
if err = json.Unmarshal(raw, &a); err == nil {
w.SeedSteps(a.Stands)
logf("steps: %d open steps the runner holds are watched from this start (%s)", len(a.Stands), standsVerb)
return
}
}
logf("steps: %s cannot be asked (%v), so a step open before this start is timed only once an event of it is heard", standsVerb, err)
}
// seedVerdicts asks the indexer's verdicts and raises the findings they hold (design 47 G §500): a verdict said while the watchman was
// down is then not lost. An indexer that does not answer is said: the findings are then raised from the verdicts heard after.
func seedVerdicts(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) {
@@ -55,6 +55,9 @@ func (w *Watchman) StepProgress(seat, node string, body []byte) {
w.say("a step-progress of %s does not read, left: %v", seat, err)
return
}
w.mu.Lock()
w.stepsHeardOf(e.Ask, e.At)
w.mu.Unlock()
w.askClock(seat, node, e.Ask, e.At)
}
@@ -75,6 +78,7 @@ func (w *Watchman) AskEnded(seat string, body []byte) {
}
w.mu.Lock()
defer w.mu.Unlock()
w.stepsHeardOf(e.Ask, e.At)
w.moved(bucket, e.Ask)
}
@@ -69,7 +69,7 @@ func (w *Watchman) timeHead(branch string) {
Headline: "A change has started no pipeline",
Explanation: "The head of the watched branch " + branch + " (source revision " + fmt.Sprint(h.revision) + ") moved " +
HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.",
Needs: "the runner to start the pipeline (its pipeline_stands and list tools say where it is)"}
Needs: "the runner to start the pipeline (its verbs mesh-pipeline-runner.stands and mesh-pipeline-runner.list say where it is)"}
left := HeadBound - w.now().Sub(h.at)
if left > 0 {
w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f})
@@ -132,7 +132,7 @@ func (w *Watchman) StepEvent(body []byte) {
w.say("a step event does not read, left: %v", err)
return
}
id := e.Pipeline + "/" + e.Result.Step
id := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact) // design 47 G A-469o
w.mu.Lock()
defer w.mu.Unlock()
for seat, q := range w.queued {
@@ -0,0 +1,217 @@
package watch
import (
"encoding/json"
"fmt"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// The watchman times every step that reads a run (design 47 G §731, §732, A-468j, A-469, A-469o, A-469p): each step started and
// not ended is kept in memory under `<pipeline>/<step>/<artifact>` (the artifact empty for a module's step), its clock begun at its
// `step-started` and begun again at each `mesh-wants` `step-progress` whose ask is the step's own ask or one its `shared` names, at
// each build seat `step-progress` of such an ask, and at the `step-ended` of such an ask. A clock past the five-minute progress
// bound raises `pipeline-stands`, cleared at the step's own `step-ended` or its next sign of life. A step waiting on a person or
// queued for a build node (§86) is owed no progress. At its start the watchman seeds its open steps from the runner's `stands`.
// stepRow is the signals table's row of a step standing still.
const stepRow = "step-stands"
// openStep is a step started and not ended.
type openStep struct {
pipeline, step, artifact string
// asks are the step's own ask and the asks its shared runs name; ask the one last heard of.
asks map[string]bool
ask string
// at is when the clock began; owed says whether the step is owed progress.
at time.Time
owed bool
}
// stepKey is the key an open step is kept under.
func stepKey(pipeline, step, artifact string) string { return pipeline + "/" + step + "/" + artifact }
// StepFindingID is the finding's id of an open step.
func StepFindingID(key string) string { return "step-" + unsafeKey.ReplaceAllString(key, "_") }
// StepStarted hears the runner's `step-started`: the step is open and its clock begins at the event's time, unless it waits on a
// person or for a build node, which are owed no progress. One that is no running or waiting step is ended.
func (w *Watchman) StepStarted(body []byte) {
var e meshpipelinerunner.StepEvent
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
w.say("a step-started does not read, left: %v", err)
return
}
key := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact)
w.mu.Lock()
defer w.mu.Unlock()
switch e.Result.Outcome {
case meshpipelinerunner.StepResultOutcomeRunning, "", meshpipelinerunner.StepResultOutcomeWaiting,
meshpipelinerunner.StepResultOutcomeWaitingForABuildNode:
default:
w.closeStep(key)
return
}
at := w.now()
switch {
case e.At != nil && !e.At.IsZero():
at = *e.At
case e.Result.Started != nil && !e.Result.Started.IsZero():
at = *e.Result.Started
}
owed := e.Result.Outcome == meshpipelinerunner.StepResultOutcomeRunning || e.Result.Outcome == ""
w.openStep(key, e.Pipeline, e.Result.Step, e.Result.Artifact, e.Result.Ask, sharedAsks(e.Result.Shared), at, owed)
}
// StepEnded hears the runner's `step-ended`: the step's clock ends and its finding is cleared.
func (w *Watchman) StepEnded(body []byte) {
var e meshpipelinerunner.StepEvent
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
w.say("a step-ended does not read, left: %v", err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
w.closeStep(stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact))
}
// WantsStepProgress hears the seat mesh-wants' `step-progress` (the assigner's, of a composition step): a step whose ask it is, or
// a shared ask it names, is alive; the runs it names are kept as the step's.
func (w *Watchman) WantsStepProgress(body []byte) {
var e meshwants.StepProgressEvent
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
w.say("a step-progress of mesh-wants does not read, left: %v", err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
var named []string
for _, s := range e.Shared {
if s.Ask != "" {
named = append(named, s.Ask)
}
}
for key, s := range w.steps {
if !s.asks[e.Ask] {
continue
}
for _, a := range named {
s.asks[a] = true
}
w.stepAlive(key, e.Ask, e.At)
}
}
// stepsHeardOf says the steps that name ask are alive: a build seat said its progress or its end (§732).
func (w *Watchman) stepsHeardOf(ask string, at time.Time) {
if ask == "" {
return
}
for key, s := range w.steps {
if s.asks[ask] {
w.stepAlive(key, ask, at)
}
}
}
// stepAlive begins an owed step's clock again at at, never earlier than it began, w.mu held.
func (w *Watchman) stepAlive(key, ask string, at time.Time) {
s := w.steps[key]
if at.IsZero() {
at = w.now()
}
s.ask = ask
if at.After(s.at) {
s.at = at
}
w.timeStep(key)
}
func sharedAsks(shared []meshpipelinerunner.Shared) []string {
var out []string
for _, s := range shared {
if s.Ask != "" {
out = append(out, s.Ask)
}
}
return out
}
// openStep keeps a step and times it, w.mu held. A step kept before keeps the asks it knew.
func (w *Watchman) openStep(key, pipeline, step, artifact, ask string, shared []string, at time.Time, owed bool) {
s := w.steps[key]
if s == nil {
s = &openStep{pipeline: pipeline, step: step, artifact: artifact, asks: map[string]bool{}}
w.steps[key] = s
}
s.at, s.owed = at, owed
if ask != "" {
s.asks[ask], s.ask = true, ask
}
for _, a := range shared {
s.asks[a] = true
}
w.timeStep(key)
}
// closeStep ends a step, w.mu held.
func (w *Watchman) closeStep(key string) {
if _, ok := w.steps[key]; !ok {
return
}
delete(w.steps, key)
id := StepFindingID(key)
w.unsignal(stepRow, id)
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
}
// timeStep times an open step, w.mu held: the progress bound from its clock, the finding raised at once when that has passed.
func (w *Watchman) timeStep(key string) {
s := w.steps[key]
id := StepFindingID(key)
w.unsignal(stepRow, id)
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
if !s.owed {
return
}
name := s.pipeline + " " + s.step
if s.artifact != "" {
name += " " + s.artifact
}
last := ""
if s.ask != "" {
last = ", last heard of its ask " + s.ask
}
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: stepRow, Since: s.at,
Summary: fmt.Sprintf("the step %s has said nothing since %s%s", name, stamp(s.at), last),
Said: "no progress in " + w.progressBound.String(),
Headline: "A pipeline step has stopped saying it lives",
Explanation: "The step " + name + " started and has not ended, and neither it nor the run it reads has said its progress for " +
w.progressBound.String() + " or more" + last + ".",
Needs: "the part that runs it to say its progress, or the step ended (the runner's verbs mesh-pipeline-runner.stands and mesh-pipeline-runner.cancel)"}
if left := w.progressBound - w.now().Sub(s.at); left > 0 {
w.signal(stepRow, id, left, signalDue{Do: dueFind, Finding: f})
return
}
w.find(f)
}
// SeedSteps seeds the open steps from the runner's `stands` at the watchman's start (§731, A-468j): each step not already kept
// begins its clock now, and a step that waits on a person or for a build node is owed none.
func (w *Watchman) SeedSteps(stands []meshpipelinerunner.Wait) {
w.mu.Lock()
defer w.mu.Unlock()
for _, s := range stands {
if s.Pipeline == "" || s.Step == "" {
continue
}
key := stepKey(s.Pipeline, s.Step, s.Artifact)
if _, ok := w.steps[key]; ok {
continue
}
owed := s.On != meshpipelinerunner.WaitOnPerson && s.On != meshpipelinerunner.WaitOnBuildNode && s.Who == ""
w.openStep(key, s.Pipeline, s.Step, s.Artifact, s.Ask, sharedAsks(s.Shared), w.now(), owed)
}
}
@@ -0,0 +1,130 @@
package watch
import (
"encoding/json"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
func stepBody(pipeline, step, artifact, ask, outcome string, at time.Time, shared ...string) []byte {
var sh []meshpipelinerunner.Shared
for _, a := range shared {
sh = append(sh, meshpipelinerunner.Shared{Run: "r", Ask: a})
}
b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c", At: &at,
Result: meshpipelinerunner.StepResult{Step: step, Artifact: artifact, Ask: ask, Outcome: outcome, Shared: sh}})
return b
}
func wantsProgress(ask string, at time.Time, shared ...string) []byte {
var sh []meshwants.Shared
for _, a := range shared {
sh = append(sh, meshwants.Shared{Machine: "n1", Run: "r", Ask: a})
}
b, _ := json.Marshal(meshwants.StepProgressEvent{Ask: ask, Asker: "a", At: at, Phase: "run", Shared: sh})
return b
}
// design 47 G §731, §732, A-468j, A-469, A-469o, A-469p (test 108's and 109's watchman case): a step started and not ended is
// timed from its `step-started` and from every sign of life of its own ask or an ask its `shared` names, raises `pipeline-stands`
// at the five-minute bound and clears at its `step-ended`; a step on a person or queued for a build node is owed none; the open
// steps are seeded from `stands`.
func TestAStepThatReadsARunIsTimed(t *testing.T) {
run := meshpipelinerunner.StepResultOutcomeRunning
key := func(pipeline, step, artifact string) string {
return signalKey(stepRow, StepFindingID(stepKey(pipeline, step, artifact)))
}
finding := func(pipeline, step, artifact string) string {
return FindingKey(ScopeDelivery, StepFindingID(stepKey(pipeline, step, artifact)), KindPipelineStands)
}
cases := []struct {
name string
run func(w *Watchman, o *fakeOut, c *clock)
// stands are the finding keys that stand at the end.
stands []string
}{
{"no progress for five minutes", func(w *Watchman, o *fakeOut, c *clock) {
w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t))
if o.ttls[key("p", "composition", "")] != ProgressBound || len(o.findings) != 0 {
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
}
expire(w, c, key("p", "composition", ""), ProgressBound)
}, []string{finding("p", "composition", "")}},
{"the assigner's progress of its ask begins the clock again, and names the runs", func(w *Watchman, o *fakeOut, c *clock) {
w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t))
c.t = c.t.Add(4 * time.Minute)
w.WantsStepProgress(wantsProgress("a1", c.t, "run1"))
if o.ttls[key("p", "composition", "")] != ProgressBound {
t.Fatalf("clock not begun again: %v", o.ttls)
}
c.t = c.t.Add(4 * time.Minute)
w.StepProgress("node-build-go", "n1", askBody("run1", c.t)) // an ask the progress named
if o.ttls[key("p", "composition", "")] != ProgressBound {
t.Fatalf("a named run's progress is the step's: %v", o.ttls)
}
w.WantsStepProgress(wantsProgress("other", c.t))
}, nil},
{"the progress of a shared run's ask, and its end, begin a test step's clock again", func(w *Watchman, o *fakeOut, c *clock) {
w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t, "run-ask"))
c.t = c.t.Add(4 * time.Minute)
w.StepProgress("node-build-go", "n1", askBody("run-ask", c.t))
c.t = c.t.Add(4 * time.Minute)
w.AskEnded("node-build-go", askBody("run-ask", c.t))
expire(w, c, key("p", "unit", "m:program"), ProgressBound)
}, []string{finding("p", "unit", "m:program")}},
{"the artifact keys the step: the other artifact's end clears only its own", func(w *Watchman, o *fakeOut, c *clock) {
w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t))
w.StepStarted(stepBody("p", "unit", "m:tools", "a3", run, c.t))
c.t = c.t.Add(ProgressBound)
w.Pulse(key("p", "unit", "m:program"), Expired, c.t)
w.Pulse(key("p", "unit", "m:tools"), Expired, c.t)
w.StepEnded(stepBody("p", "unit", "m:tools", "a3", meshpipelinerunner.StepResultOutcomePassed, c.t))
}, []string{finding("p", "unit", "m:program")}},
{"its step-ended clears the finding", func(w *Watchman, o *fakeOut, c *clock) {
w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t))
expire(w, c, key("p", "composition", ""), ProgressBound)
w.StepEnded(stepBody("p", "composition", "", "a1", meshpipelinerunner.StepResultOutcomeFailed, c.t))
}, nil},
{"a step on a person or queued for a build node is owed none", func(w *Watchman, o *fakeOut, c *clock) {
w.StepStarted(stepBody("p", "approval", "", "", meshpipelinerunner.StepResultOutcomeWaiting, c.t))
w.StepStarted(stepBody("p", "unit", "m:program", "a2", meshpipelinerunner.StepResultOutcomeWaitingForABuildNode, c.t))
c.t = c.t.Add(time.Hour)
if len(o.pulses) != 0 || len(o.findings) != 0 {
t.Fatalf("owed none: %v %v", o.pulses, o.findings)
}
w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t)) // taken: now owed
expire(w, c, key("p", "unit", "m:program"), ProgressBound)
}, []string{finding("p", "unit", "m:program")}},
{"seeded from stands: each clock begins at the start, a step on a person is owed none", func(w *Watchman, o *fakeOut, c *clock) {
w.SeedSteps([]meshpipelinerunner.Wait{
{Pipeline: "p", Step: "composition", Ask: "a1", On: ""},
{Pipeline: "p", Step: "unit", Artifact: "m:program", Shared: []meshpipelinerunner.Shared{{Run: "r", Ask: "run-ask"}}},
{Pipeline: "p", Step: "approval", On: meshpipelinerunner.WaitOnPerson},
})
if len(o.findings) != 0 || o.ttls[key("p", "composition", "")] != ProgressBound || len(o.pulses) != 2 {
t.Fatalf("seeded at the start: %v %v", o.ttls, o.findings)
}
c.t = c.t.Add(4 * time.Minute)
w.WantsStepProgress(wantsProgress("a1", c.t)) // the composition step lives
expire(w, c, key("p", "unit", "m:program"), time.Minute)
}, []string{finding("p", "unit", "m:program")}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
w, o, c := setup()
w.SetStepBound(ProgressBound) // setup shortens it for the older tests
tc.run(w, o, c)
if len(o.findings) != len(tc.stands) {
t.Fatalf("findings %v, want %v", o.findings, tc.stands)
}
for _, k := range tc.stands {
if _, ok := o.findings[k]; !ok {
t.Fatalf("findings %v, want %s", o.findings, k)
}
}
})
}
}
@@ -121,6 +121,8 @@ type Watchman struct {
recordMilestone map[string]string
// queued is, by build seat, each step queued on it for a build node and since when (pipelines.go).
queued map[string]map[string]time.Time
// steps is each pipeline step started and not ended, by `<pipeline>/<step>/<artifact>` (steps.go, design 47 G §732).
steps map[string]*openStep
// awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go).
awaySaid map[string]time.Time
// opened is each record past its bound and not ended, with its finding (records.go).
@@ -135,7 +137,7 @@ func New(module string, out Out, now func() time.Time) *Watchman {
advised: map[string]int{}, advisedSince: map[string]time.Time{},
provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{},
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, steps: map[string]*openStep{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
}
// Logf sets where a write that failed is said; it is silent until set.
@@ -127,11 +127,11 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
}
// The join: its first view asked again by hand is a hand act, recorded through the seat (views.StatusTools). The waits the
// indexer holds at the watchman's start are asked of its tool (ADR 0633 §5), a verb edge as the view's.
if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source"}; !reflect.DeepEqual(m.Invokes, want) {
if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", "seat:mesh-pipeline-runner.stands"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably.
if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) {
if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-wants.step-progress", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
// 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4).
+3 -1
View File
@@ -26,7 +26,8 @@
"seat:mesh-broker.show",
"seat:mesh-catalog.revision",
"seat:mesh-catalog.registrations",
"seat:mesh-delivery.source"
"seat:mesh-delivery.source",
"seat:mesh-pipeline-runner.stands"
],
"consumes": [
"seat:mesh-wants.views.wants",
@@ -64,6 +65,7 @@
"mesh-pipeline-runner.pipeline-started",
"mesh-pipeline-runner.step-started",
"mesh-pipeline-runner.step-ended",
"mesh-wants.step-progress",
"mesh-pipeline-runner.wait-held",
"mesh-pipeline-runner.wait-ended",
"mesh-wants.wait-held",